diff --git a/test/e2e/README.md b/test/e2e/README.md index 2fa460eea4a..37257e1cb9e 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -337,9 +337,10 @@ The host must provide the util-linux `script` command and GNU `timeout` command. The helper rebuilds the candidate CLI, runs `connect --probe-only`, and then runs two `launch` sessions during the same fixed lease. -Each pseudo-terminal session sends a unique prompt, requires the exact reply, -sends `/exit`, and requires process exit status `0`. -The helper uses exact terminal behavior instead of a wall-clock pass threshold. +Each real pseudo-terminal session sends two distinct messages and `/exit`, then +requires process exit status `0`. The OpenClaw session store must append two +nonempty `user` and `assistant` record pairs in one session. The helper does not +compare message content. Terminal output is a bounded failure diagnostic only. Deterministic unit tests separately prove selection of the complete preflight and lease paths, stale-producer exclusion, the fixed time-unsafe quarantine, refusal to recover when prior evidence cannot be durably fenced, and the named diff --git a/test/e2e/live/full-e2e.test.ts b/test/e2e/live/full-e2e.test.ts index fd66ca21cfd..df887bfd857 100644 --- a/test/e2e/live/full-e2e.test.ts +++ b/test/e2e/live/full-e2e.test.ts @@ -42,7 +42,7 @@ import { fullE2eInferenceProbeEvidence, runFullE2eInferenceProbe, } from "./full-e2e-inference-probe.ts"; -import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; +import { runOpenClawLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; import { bindApprovedPrBaseForBaseImageComparison } from "./pr-base-comparison.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-full"; @@ -141,15 +141,13 @@ async function runOpenClawLaunchTurnAfterRecovery(input: { ); expect(recovery.exitCode, resultText(recovery)).toBe(0); - await runLaunchReadinessLeaseTurns({ + await runOpenClawLaunchReadinessLeaseTurns({ artifactName: "phase-4-openclaw-launch-turn", cliCommand: USE_PREINSTALLED_LAUNCHABLE ? "nemoclaw" : process.execPath, ...(!USE_PREINSTALLED_LAUNCHABLE ? { cliEntrypoint: CLI_ENTRYPOINT } : {}), env: env(PORTABLE_PROFILE ? { DOCKER_HOST: "" } : {}), exitCommand: "/exit", host: input.host, - postReplyReadyText: "gateway connected | idle", - readyText: "gateway connected | idle", redactionValues: input.redactionValues, sandboxName: SANDBOX_NAME, }); @@ -391,7 +389,7 @@ test("full e2e: install, onboard, inference, cli operations, and cleanup", { "direct hosted inference and sandbox inference.local both respond", ...(process.platform === "linux" ? [ - "a recovered OpenClaw sandbox completes a /exit launch turn through inference.local and restores the mutable config permission contract", + "each of two PTY launches records two ordered structured turns and restores the mutable config permission contract", ] : []), "nemoclaw logs produces output and cleanup removes registry state", diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index 72453288f45..a84d3a414ae 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -370,9 +370,6 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }); expect(trustedRecovery.timedOut, resultText(trustedRecovery)).toBe(false); expect(trustedRecovery.exitCode, resultText(trustedRecovery)).toBe(0); - expect(resultText(trustedRecovery)).toMatch( - /Probe complete: (?:recovered OpenClaw gateway|OpenClaw gateway is running)/, - ); const restartStateLockPlan = await sandbox.exec( instance.sandboxName, ["python3", "-c", OPENCLAW_STATE_LOCK_PLAN_PROBE], @@ -501,7 +498,6 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }); expect(legacyRecovery.timedOut, resultText(legacyRecovery)).toBe(false); expect(legacyRecovery.exitCode, resultText(legacyRecovery)).toBe(0); - expect(resultText(legacyRecovery)).toContain("Probe complete: recovered OpenClaw gateway"); const legacyStateLockPlan = await sandbox.exec( instance.sandboxName, ["python3", "-c", OPENCLAW_STATE_LOCK_PLAN_PROBE], diff --git a/test/e2e/live/hermes-e2e-phases.ts b/test/e2e/live/hermes-e2e-phases.ts index 3813ef4e55f..05fcd6d440d 100644 --- a/test/e2e/live/hermes-e2e-phases.ts +++ b/test/e2e/live/hermes-e2e-phases.ts @@ -5,7 +5,7 @@ export const HERMES_E2E_PHASES = [ "prepare clean Hermes runner", "install and onboard Hermes sandbox", "validate sandbox layout, health, and skill activation", - "restart Hermes gateway, validate supervision, and complete two launch turns", + "restart Hermes gateway and validate supervision", "exercise hosted and inference.local routes", "validate CLI manifest and locked-config behavior", "finalize Hermes sandbox resources", diff --git a/test/e2e/live/hermes-e2e.test.ts b/test/e2e/live/hermes-e2e.test.ts index 2c4e6a1102d..769ac4ba513 100644 --- a/test/e2e/live/hermes-e2e.test.ts +++ b/test/e2e/live/hermes-e2e.test.ts @@ -24,7 +24,6 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { assertHermesCliAdapterLiveContract, stripAnsi } from "./hermes-cli-adapter-live.ts"; import { HERMES_E2E_PHASES } from "./hermes-e2e-phases.ts"; import { assertHermesSkillLifecycle } from "./hermes-skill-lifecycle.ts"; -import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; import { expectPackageDatabaseReadOnly } from "./package-database-read-only.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-hermes"; @@ -599,7 +598,7 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(httpStatusOk(dashboardInternal.stdout)).toBe(true); } - progress.phase("restart Hermes gateway, validate supervision, and complete two launch turns"); + progress.phase("restart Hermes gateway and validate supervision"); // Phase 5: host-mediated Hermes gateway restart. This validates the // runtime contract behind #2426 against a real OpenShell/Hermes sandbox: // The installed supervision tree controls the gateway process, direct @@ -1256,17 +1255,9 @@ test("hermes-e2e: install.sh onboards Hermes and proves health plus live inferen expect(routingTopologyCaptures).toBe(2); - await (process.platform === "linux" - ? runLaunchReadinessLeaseTurns({ - artifactName: "phase-5-hermes-launch-turn-after-recovery", - cliCommand: "nemoclaw", - env, - host, - redactionValues, - sandboxName: SANDBOX_NAME, - }) - : Promise.resolve()); - + // OpenClaw launch qualification now reads its structured JSONL session + // store. Hermes owns a different SQLite contract, so this target must not + // infer Hermes replies from terminal copy through the OpenClaw helper. progress.phase("exercise hosted and inference.local routes"); // Phase 6: live inference through both the external provider and the // sandbox's inference.local route. diff --git a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts index 356c1bc13d0..2b1aada179a 100644 --- a/test/e2e/live/issue-2478-crash-loop-recovery.test.ts +++ b/test/e2e/live/issue-2478-crash-loop-recovery.test.ts @@ -229,7 +229,7 @@ async function runProbeOnly( }, sandboxName: string, artifactName: string, -): Promise<"connect" | "supervisor"> { +): Promise { const result = await host.nemoclaw([sandboxName, "connect", "--probe-only"], { artifactName, env: probeEnv(), @@ -239,18 +239,6 @@ async function runProbeOnly( result.exitCode, `${artifactName} failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, ).toBe(0); - const connectRecovery = `Probe complete: recovered OpenClaw gateway in '${sandboxName}'.`; - const supervisorRecovery = `Probe complete: OpenClaw gateway is running in '${sandboxName}'.`; - const recoveryPath = result.stdout.includes(connectRecovery) - ? "connect" - : result.stdout.includes(supervisorRecovery) - ? "supervisor" - : null; - expect( - recoveryPath, - `${artifactName} did not observe a healthy gateway after termination\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, - ).not.toBeNull(); - return recoveryPath!; } async function terminateGatewayIdentity( @@ -369,7 +357,7 @@ test("gateway recovery restores the guard chain and keeps the recovered process preRecoveryIdentity!, "functional-recovery-terminate-gateway", ); - const recoveryPath = await runProbeOnly( + await runProbeOnly( host, instance.sandboxName, "functional-recovery-connect-probe-only", @@ -398,7 +386,6 @@ test("gateway recovery restores the guard chain and keeps the recovered process await artifacts.writeJson("functional-recovery-summary.json", { initialIdentity, preRecoveryIdentity, - recoveryPath, recoveredIdentity, stableIdentity, stabilitySeconds: STABILITY_SECONDS, diff --git a/test/e2e/live/launch-agent-turn.ts b/test/e2e/live/launch-agent-turn.ts index 21064d82114..33bc1c5184a 100644 --- a/test/e2e/live/launch-agent-turn.ts +++ b/test/e2e/live/launch-agent-turn.ts @@ -1,15 +1,186 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { createHash } from "node:crypto"; +import { randomUUID } from "node:crypto"; import { resultText } from "../fixtures/clients/command.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -const EXPECTED_REPLY = "PONG"; -const PROMPT = - "Reply with the one uppercase word formed by the letters P, O, N, G in that order. Do not use tools."; +// OpenClaw owns the JSONL session store and does not expose a structured +// result from `nemoclaw launch`. This verifier records an in-sandbox baseline, +// then qualifies only complete user and assistant records appended after that +// baseline. Session content never moves to the host. +export const OPENCLAW_SESSION_EVIDENCE_SCRIPT = String.raw` +const crypto = require("node:crypto"); +const fs = require("node:fs"); +const path = require("node:path"); + +const [mode, sessionRoot, baselinePath, expectedTurnsText] = process.argv.slice(1); + +function finish(exitCode, reason, detail = {}) { + if (reason) process.stderr.write(JSON.stringify({ reason, ...detail }) + "\n"); + process.exit(exitCode); +} + +function completeOffset(raw) { + return raw.endsWith("\n") ? raw.length : raw.lastIndexOf("\n") + 1; +} + +function digest(value) { + return crypto.createHash("sha256").update(value).digest("hex"); +} + +function sessionFileNames() { + try { + return fs + .readdirSync(sessionRoot) + .filter((name) => name.endsWith(".jsonl") && !name.endsWith(".trajectory.jsonl")) + .sort(); + } catch (error) { + if (error && error.code === "ENOENT") return []; + finish(2, "session_store_unreadable"); + } +} + +function readCompleteSession(fileName) { + let raw; + try { + raw = fs.readFileSync(path.join(sessionRoot, fileName), "utf8"); + } catch { + finish(2, "session_unreadable", { sessionId: fileName.slice(0, -6) }); + } + const offset = completeOffset(raw); + return { offset, complete: raw.slice(0, offset), raw }; +} + +function recordBaseline() { + const sessions = {}; + for (const fileName of sessionFileNames()) { + const { offset, complete } = readCompleteSession(fileName); + sessions[fileName] = { offset, digest: digest(complete) }; + } + try { + fs.writeFileSync( + baselinePath, + JSON.stringify({ schemaVersion: 1, sessions }), + { encoding: "utf8", flag: "wx", mode: 0o600 }, + ); + } catch { + finish(2, "baseline_write_failed"); + } + finish(0); +} + +function readBaseline() { + let value; + try { + value = JSON.parse(fs.readFileSync(baselinePath, "utf8")); + } catch { + finish(2, "baseline_unreadable"); + } + if ( + !value || + value.schemaVersion !== 1 || + !value.sessions || + typeof value.sessions !== "object" || + Array.isArray(value.sessions) + ) { + finish(2, "baseline_invalid"); + } + for (const entry of Object.values(value.sessions)) { + if ( + !entry || + typeof entry !== "object" || + !Number.isSafeInteger(entry.offset) || + entry.offset < 0 || + typeof entry.digest !== "string" || + !/^[0-9a-f]{64}$/.test(entry.digest) + ) { + finish(2, "baseline_invalid"); + } + } + return value.sessions; +} + +function hasStructuredContent(message) { + if (typeof message.content === "string") return message.content.length > 0; + return Array.isArray(message.content) && message.content.length > 0; +} + +function appendedMessages(fileName, baseline) { + const { offset, complete, raw } = readCompleteSession(fileName); + const prior = baseline[fileName]; + const priorOffset = prior ? prior.offset : 0; + if (raw.length !== offset) { + finish(2, "session_record_incomplete", { sessionId: fileName.slice(0, -6) }); + } + if (offset < priorOffset) finish(2, "session_truncated", { sessionId: fileName.slice(0, -6) }); + if (prior && digest(raw.slice(0, priorOffset)) !== prior.digest) { + finish(2, "session_rewritten", { sessionId: fileName.slice(0, -6) }); + } + + const messages = []; + for (const line of complete.slice(priorOffset).split("\n")) { + if (!line.trim()) continue; + let record; + try { + record = JSON.parse(line); + } catch { + finish(2, "malformed_session", { sessionId: fileName.slice(0, -6) }); + } + if (!record || record.type !== "message" || !record.message) continue; + const role = record.message.role; + if (role !== "user" && role !== "assistant") continue; + messages.push({ role, hasStructuredContent: hasStructuredContent(record.message) }); + } + return messages; +} + +function qualifyTurns() { + const expectedTurns = Number(expectedTurnsText); + if (!Number.isSafeInteger(expectedTurns) || expectedTurns < 1) { + finish(2, "expected_turn_count_invalid"); + } + + const baseline = readBaseline(); + const currentFiles = sessionFileNames(); + for (const fileName of Object.keys(baseline)) { + if (!currentFiles.includes(fileName)) { + finish(2, "session_removed", { sessionId: fileName.slice(0, -6) }); + } + } + + const changedSessions = currentFiles + .map((fileName) => ({ + sessionId: fileName.slice(0, -6), + messages: appendedMessages(fileName, baseline), + })) + .filter((session) => session.messages.length > 0); + if (changedSessions.length === 0) finish(1); + if (changedSessions.length > 1) finish(2, "multiple_sessions_changed"); + + const { messages, sessionId } = changedSessions[0]; + const expectedRoles = Array.from({ length: expectedTurns }, () => ["user", "assistant"]).flat(); + for (const [index, message] of messages.entries()) { + if (index >= expectedRoles.length) finish(2, "extra_message", { sessionId }); + if (message.role !== expectedRoles[index]) { + finish(2, "message_order_invalid", { sessionId }); + } + if (!message.hasStructuredContent) finish(2, "message_content_empty", { sessionId }); + } + if (messages.length < expectedRoles.length) finish(1); + finish(0); +} + +try { + if (mode === "baseline") recordBaseline(); + if (mode === "qualify") qualifyTurns(); +} catch { + finish(2, "verifier_failed"); +} +finish(2, "mode_invalid"); +`; export const LAUNCH_TURN_SCRIPT = String.raw`set -euo pipefail command -v script >/dev/null 2>&1 @@ -17,10 +188,22 @@ command -v timeout >/dev/null 2>&1 session_dir="$(mktemp -d /tmp/nemoclaw-launch-turn.XXXXXX)" capture="$session_dir/terminal.log" +driver_error="$session_dir/pty-driver.err" +evidence_error="$session_dir/session-evidence.err" input="$session_dir/input" +baseline_path="/tmp/nemoclaw-launch-session-$NEMOCLAW_LAUNCH_RUN_ID.json" session_pid="" +remove_session_baseline() { + "$NEMOCLAW_OPENSHELL_COMMAND" sandbox exec \ + --name "$NEMOCLAW_LAUNCH_SANDBOX" -- \ + rm -f -- "$baseline_path" +} + cleanup() { + local original_status=$? + local cleanup_status=0 + trap - EXIT exec 3>&- 2>/dev/null || true if [[ -n "$session_pid" ]] && kill -0 "$session_pid" 2>/dev/null; then kill -TERM "$session_pid" 2>/dev/null || true @@ -30,10 +213,77 @@ cleanup() { if [[ -n "$session_pid" ]]; then wait "$session_pid" 2>/dev/null || true fi + if ! remove_session_baseline >/dev/null 2>&1; then + echo "structured session baseline cleanup failed" >&2 + cleanup_status=1 + fi rm -rf -- "$session_dir" + if [[ "$original_status" != 0 ]]; then + exit "$original_status" + fi + exit "$cleanup_status" } trap cleanup EXIT +terminal_diagnostic() { + if [[ -f "$capture" ]]; then + echo "bounded terminal diagnostic (last 4096 bytes):" >&2 + tail -c 4096 "$capture" >&2 || true + fi + if [[ -s "$driver_error" ]]; then + echo "bounded PTY driver diagnostic (last 2048 bytes):" >&2 + tail -c 2048 "$driver_error" >&2 || true + fi +} + +fail_launch_session() { + echo "$1" >&2 + if [[ -s "$evidence_error" ]]; then + tail -c 2048 "$evidence_error" >&2 || true + fi + terminal_diagnostic + exit 1 +} + +session_evidence() { + local mode="$1" + local expected_turns="" + if [[ "$#" -gt 1 ]]; then + expected_turns="$2" + fi + "$NEMOCLAW_OPENSHELL_COMMAND" sandbox exec \ + --name "$NEMOCLAW_LAUNCH_SANDBOX" -- \ + node -e "$NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT" \ + "$mode" \ + "$NEMOCLAW_LAUNCH_SESSION_ROOT" \ + "$baseline_path" \ + "$expected_turns" +} + +wait_for_turn_count() { + local expected_turns="$1" + local evidence_status + for _ in {1..180}; do + if session_evidence qualify "$expected_turns" >/dev/null 2>"$evidence_error"; then + return 0 + else + evidence_status=$? + fi + if [[ "$evidence_status" != 1 ]]; then + fail_launch_session "structured session evidence was invalid or unavailable (status $evidence_status)" + fi + if ! kill -0 "$session_pid" 2>/dev/null; then + break + fi + sleep 1 + done + fail_launch_session "launch did not record the required structured session turns" +} + +if ! session_evidence baseline >/dev/null 2>"$evidence_error"; then + fail_launch_session "launch could not record the structured session baseline" +fi + mkfifo -m 600 "$input" if [[ -n "$NEMOCLAW_LAUNCH_ENTRYPOINT" ]]; then printf -v launch_command '%q %q %q %q' \ @@ -45,7 +295,8 @@ else fi timeout --kill-after=5s 250s \ - script --quiet --return --flush --command "$launch_command" "$capture" <"$input" & + script --quiet --return --flush --command "$launch_command" "$capture" \ + <"$input" >/dev/null 2>"$driver_error" & session_pid=$! exec 3>"$input" @@ -61,94 +312,13 @@ for _ in {1..100}; do sleep 0.1 done if [[ "$capture_ready" != 1 ]]; then - echo "launch did not create a terminal capture" >&2 - exit 1 + fail_launch_session "launch did not create a PTY diagnostic capture" fi -if [[ -n "$NEMOCLAW_LAUNCH_READY_TEXT" ]]; then - ready_seen=0 - for _ in {1..60}; do - if grep -Fq -- "$NEMOCLAW_LAUNCH_READY_TEXT" "$capture"; then - ready_seen=1 - break - fi - if ! kill -0 "$session_pid" 2>/dev/null; then - break - fi - sleep 1 - done - if [[ "$ready_seen" != 1 ]]; then - echo "launch did not reach the expected TUI state" >&2 - exit 1 - fi -else - # Hermes accepts buffered terminal input after its startup render settles. - sleep 12 -fi -response_start="$(wc -c <"$capture")" -printf '%s\r' "$NEMOCLAW_LAUNCH_PROMPT" >&3 - -reply_seen=0 -normalized_response() { - tail -c "+$((response_start + 1))" "$capture" \ - | sed -E $'s/\x1B][^\x07\x1B]*(\x07|\x1B\\\\)//g' \ - | sed -E $'s|\x1B\\[[0-?]*[ -/]*[@-~]||g' \ - | tr '\r' '\n' \ - | LC_ALL=C tr -d '\000-\010\013\014\016-\037\177' -} -has_exact_reply() { - normalized_response | awk -v expected="$NEMOCLAW_LAUNCH_EXPECTED_REPLY" ' - { - line = $0 - sub(/^[[:space:]]+/, "", line) - sub(/[[:space:]]+$/, "", line) - if (line == expected) found = 1 - } - END { exit found ? 0 : 1 } - ' -} -has_post_reply_ready() { - normalized_response | awk \ - -v expected="$NEMOCLAW_LAUNCH_EXPECTED_REPLY" \ - -v ready="$NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT" ' - { - line = $0 - sub(/^[[:space:]]+/, "", line) - sub(/[[:space:]]+$/, "", line) - if (line == expected) reply = 1 - if (reply && line == ready) found = 1 - } - END { exit found ? 0 : 1 } - ' -} -for _ in {1..180}; do - if has_exact_reply; then - reply_seen=1 - break - fi - if ! kill -0 "$session_pid" 2>/dev/null; then - break - fi - sleep 1 -done - -if [[ "$reply_seen" = 1 && -n "$NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT" ]]; then - post_reply_ready_seen=0 - for _ in {1..60}; do - if has_post_reply_ready; then - post_reply_ready_seen=1 - break - fi - if ! kill -0 "$session_pid" 2>/dev/null; then - break - fi - sleep 1 - done - if [[ "$post_reply_ready_seen" != 1 ]]; then - echo "launch did not return to the expected TUI state after the reply" >&2 - exit 1 - fi -fi +printf '%s\r' "$NEMOCLAW_LAUNCH_FIRST_INPUT" >&3 +wait_for_turn_count 1 +printf '%s\r' "$NEMOCLAW_LAUNCH_SECOND_INPUT" >&3 +wait_for_turn_count 2 if [[ -n "$NEMOCLAW_LAUNCH_EXIT_COMMAND" ]]; then printf '%s\r' "$NEMOCLAW_LAUNCH_EXIT_COMMAND" >&3 @@ -163,45 +333,50 @@ else fi exec 3>&- -if [[ "$reply_seen" != 1 ]]; then - echo "launch did not produce the expected agent reply" >&2 - exit 1 -fi if wait "$session_pid"; then launch_status=0 else launch_status=$? fi session_pid="" + if [[ "$launch_status" != 0 ]]; then echo "launch exited with status $launch_status" >&2 + terminal_diagnostic exit "$launch_status" fi -printf '%s\n' "NEMOCLAW_LAUNCH_TURN_OK" +if ! remove_session_baseline >/dev/null 2>"$evidence_error"; then + fail_launch_session "launch could not remove the structured session baseline" +fi `; -export interface LaunchAgentTurnOptions { +export interface OpenClawLaunchSessionOptions { artifactName: string; cliCommand: string; cliEntrypoint?: string; env: NodeJS.ProcessEnv; exitCommand?: string; host: HostCliClient; - postReplyReadyText?: string; - readyText?: string; redactionValues: string[]; sandboxName: string; - expectedReply?: string; - prompt?: string; beforeLaunchTurns?: () => Promise | void; } -export async function runLaunchAgentTurn( - options: LaunchAgentTurnOptions, +function uniqueTurnInputs(): { first: string; second: string } { + const fragment = randomUUID().replaceAll("-", ""); + return { + first: `Reply briefly without using tools. Request identifier: ${fragment.slice(0, 16)}.`, + second: `Reply briefly again without using tools. Request identifier: ${fragment.slice(16)}.`, + }; +} + +export async function runOpenClawLaunchSession( + options: OpenClawLaunchSessionOptions, ): Promise { if (process.platform !== "linux") { - throw new Error("launch agent turn coverage requires the Linux util-linux PTY driver"); + throw new Error("launch session coverage requires the Linux util-linux PTY driver"); } + const inputs = uniqueTurnInputs(); const result = await options.host.command("bash", ["-lc", LAUNCH_TURN_SCRIPT], { artifactName: options.artifactName, env: { @@ -209,37 +384,27 @@ export async function runLaunchAgentTurn( NEMOCLAW_LAUNCH_COMMAND: options.cliCommand, NEMOCLAW_LAUNCH_ENTRYPOINT: options.cliEntrypoint ?? "", NEMOCLAW_LAUNCH_EXIT_COMMAND: options.exitCommand ?? "", - NEMOCLAW_LAUNCH_EXPECTED_REPLY: options.expectedReply ?? EXPECTED_REPLY, - NEMOCLAW_LAUNCH_PROMPT: options.prompt ?? PROMPT, - NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT: options.postReplyReadyText ?? "", - NEMOCLAW_LAUNCH_READY_TEXT: options.readyText ?? "", + NEMOCLAW_LAUNCH_FIRST_INPUT: inputs.first, + NEMOCLAW_LAUNCH_RUN_ID: randomUUID().replaceAll("-", ""), NEMOCLAW_LAUNCH_SANDBOX: options.sandboxName, + NEMOCLAW_LAUNCH_SECOND_INPUT: inputs.second, + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_LAUNCH_SESSION_ROOT: "/sandbox/.openclaw/agents/main/sessions", + NEMOCLAW_OPENSHELL_COMMAND: options.host.openshellCommandPath, TERM: "xterm-256color", }, redactionValues: options.redactionValues, timeoutMs: 280_000, }); - if (result.exitCode !== 0 || !result.stdout.includes("NEMOCLAW_LAUNCH_TURN_OK")) { - throw new Error(`launch agent turn failed: ${resultText(result)}`); + if (result.exitCode !== 0) { + throw new Error(`launch session failed: ${resultText(result)}`); } return result; } -function uniqueTurnContract(artifactName: string, ordinal: "FIRST" | "SECOND") { - const fragment = createHash("sha256") - .update(`${artifactName}:${ordinal}`) - .digest("hex") - .slice(0, 12); - const expectedReply = `NEMOCLAW_${fragment.toUpperCase()}_${ordinal}_OK`; - return { - expectedReply, - prompt: - `Join these four fragments with underscores and put only the result on its own line: ` + - `NEMOCLAW, ${fragment.toUpperCase()}, ${ordinal}, OK. Do not use tools.`, - }; -} - -export async function runLaunchReadinessLeaseTurns(options: LaunchAgentTurnOptions): Promise { +export async function runOpenClawLaunchReadinessLeaseTurns( + options: OpenClawLaunchSessionOptions, +): Promise { const probeArgs = options.cliEntrypoint ? [options.cliEntrypoint, options.sandboxName, "connect", "--probe-only"] : [options.sandboxName, "connect", "--probe-only"]; @@ -255,11 +420,10 @@ export async function runLaunchReadinessLeaseTurns(options: LaunchAgentTurnOptio await options.beforeLaunchTurns?.(); - for (const ordinal of ["FIRST", "SECOND"] as const) { - await runLaunchAgentTurn({ + for (const ordinal of ["first", "second"] as const) { + await runOpenClawLaunchSession({ ...options, - artifactName: `${options.artifactName}-${ordinal.toLowerCase()}`, - ...uniqueTurnContract(options.artifactName, ordinal), + artifactName: `${options.artifactName}-${ordinal}`, }); } } diff --git a/test/e2e/live/launch-readiness-lease-acceptance.test.ts b/test/e2e/live/launch-readiness-lease-acceptance.test.ts index 7084031cdcf..5e72c8b1098 100644 --- a/test/e2e/live/launch-readiness-lease-acceptance.test.ts +++ b/test/e2e/live/launch-readiness-lease-acceptance.test.ts @@ -4,7 +4,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { CLI_ENTRYPOINT } from "../fixtures/paths.ts"; import { readRegistrySandboxEntry } from "../fixtures/phases/index.ts"; -import { runLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; +import { runOpenClawLaunchReadinessLeaseTurns } from "./launch-agent-turn.ts"; const SANDBOX_NAME = process.env.NEMOCLAW_ACCEPTANCE_SANDBOX?.trim() ?? ""; @@ -16,11 +16,11 @@ test.runIf(process.platform === "linux" && SANDBOX_NAME.length > 0)( e2ePhases: [ "verify the existing locked-image sandbox", "produce launch-readiness evidence", - "complete two exact-reply PTY launch turns", + "complete two PTY launch sessions with structured turn evidence", ], }, }, - async ({ host, progress }) => { + async ({ host, progress, secrets }) => { progress.phase("verify the existing locked-image sandbox"); const entry = readRegistrySandboxEntry(SANDBOX_NAME); expect(entry.agent).toBe("openclaw"); @@ -31,19 +31,17 @@ test.runIf(process.platform === "linux" && SANDBOX_NAME.length > 0)( expect(workload.reference).toMatch(/@sha256:[0-9a-f]{64}$/u); expect(entry.imageTag).toBe(workload.reference); progress.phase("produce launch-readiness evidence"); - await runLaunchReadinessLeaseTurns({ + await runOpenClawLaunchReadinessLeaseTurns({ artifactName: "launch-readiness-locked-image", cliCommand: process.execPath, cliEntrypoint: CLI_ENTRYPOINT, env: process.env, exitCommand: "/exit", host, - postReplyReadyText: "gateway connected | idle", - readyText: "gateway connected | idle", - redactionValues: [], + redactionValues: secrets.redactionValues(), sandboxName: SANDBOX_NAME, beforeLaunchTurns: () => { - progress.phase("complete two exact-reply PTY launch turns"); + progress.phase("complete two PTY launch sessions with structured turn evidence"); }, }); }, diff --git a/test/e2e/support/launch-agent-turn.test.ts b/test/e2e/support/launch-agent-turn.test.ts index a41a5259ec0..6f05442b73d 100644 --- a/test/e2e/support/launch-agent-turn.test.ts +++ b/test/e2e/support/launch-agent-turn.test.ts @@ -2,94 +2,408 @@ // SPDX-License-Identifier: Apache-2.0 import { spawnSync } from "node:child_process"; -import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + appendFileSync, + chmodSync, + existsSync, + mkdtempSync, + mkdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { basename, join } from "node:path"; import { expect, it } from "vitest"; -import { LAUNCH_TURN_SCRIPT, runLaunchReadinessLeaseTurns } from "../live/launch-agent-turn.ts"; +import { + LAUNCH_TURN_SCRIPT, + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + runOpenClawLaunchReadinessLeaseTurns, +} from "../live/launch-agent-turn.ts"; -function runLaunchTurnFixture(exitStatus: number, reply = "PONG", closeAfterReply = false) { +type SessionRecords = Record; +type FixtureMode = + | "cleanup-failure" + | "invalid-order" + | "nonzero" + | "nonzero-cleanup-failure" + | "valid"; + +function message(role: "assistant" | "user", content = "nonempty"): string { + return JSON.stringify({ + message: { content: [{ text: content, type: "text" }], role }, + type: "message", + }); +} + +function emptyMessage(role: "assistant" | "user"): string { + return JSON.stringify({ message: { content: [], role }, type: "message" }); +} + +function writeSessionRecords( + root: string, + sessions: SessionRecords, + append: boolean, + finalNewline = true, +): void { + for (const [sessionId, records] of Object.entries(sessions)) { + const filePath = join(root, `${sessionId}.jsonl`); + const body = records.length > 0 ? `${records.join("\n")}${finalNewline ? "\n" : ""}` : ""; + const writeRecords = append ? appendFileSync : writeFileSync; + writeRecords(filePath, body); + } +} + +function runEvidenceFixture(input: { + after: SessionRecords; + afterFinalNewline?: boolean; + before?: SessionRecords; + expectedTurns: number; +}) { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-evidence-")); + const baselinePath = join(fixtureRoot, "baseline.json"); + const sessionRoot = join(fixtureRoot, "sessions"); + mkdirSync(sessionRoot); + try { + writeSessionRecords(sessionRoot, input.before ?? {}, false); + const baseline = spawnSync( + process.execPath, + ["-e", OPENCLAW_SESSION_EVIDENCE_SCRIPT, "baseline", sessionRoot, baselinePath, ""], + { encoding: "utf8" }, + ); + writeSessionRecords(sessionRoot, input.after, true, input.afterFinalNewline ?? true); + const qualification = spawnSync( + process.execPath, + [ + "-e", + OPENCLAW_SESSION_EVIDENCE_SCRIPT, + "qualify", + sessionRoot, + baselinePath, + String(input.expectedTurns), + ], + { encoding: "utf8" }, + ); + return { baseline, baselineMode: statSync(baselinePath).mode & 0o777, qualification }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +} + +function runBaselineMutationFixture(mutation: "invalid" | "removed" | "rewritten" | "truncated") { + const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-baseline-")); + const baselinePath = join(fixtureRoot, "baseline.json"); + const sessionRoot = join(fixtureRoot, "sessions"); + const sessionPath = join(sessionRoot, "session-a.jsonl"); + mkdirSync(sessionRoot); + writeSessionRecords(sessionRoot, { "session-a": [message("user"), message("assistant")] }, false); + try { + const baseline = spawnSync( + process.execPath, + ["-e", OPENCLAW_SESSION_EVIDENCE_SCRIPT, "baseline", sessionRoot, baselinePath, ""], + { encoding: "utf8" }, + ); + const applyMutation: Record void> = { + invalid: () => writeFileSync(baselinePath, "{}"), + removed: () => rmSync(sessionPath), + rewritten: () => + writeFileSync( + sessionPath, + readFileSync(sessionPath, "utf8").replace("nonempty", "changed!"), + ), + truncated: () => writeFileSync(sessionPath, ""), + }; + applyMutation[mutation](); + const qualification = spawnSync( + process.execPath, + ["-e", OPENCLAW_SESSION_EVIDENCE_SCRIPT, "qualify", sessionRoot, baselinePath, "1"], + { encoding: "utf8" }, + ); + return { baseline, qualification }; + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +} + +function runLaunchSessionFixture(mode: FixtureMode, terminalCopy: "ansi" | "plain") { const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-turn-")); - const scriptStub = join(fixtureRoot, "script"); - const sleepStub = join(fixtureRoot, "sleep"); - const timeoutStub = join(fixtureRoot, "timeout"); + const fakeLaunch = join(fixtureRoot, "fake-launch.cjs"); + const fakeOpenshell = join(fixtureRoot, "openshell"); + const sessionRoot = join(fixtureRoot, "sessions"); + const ttyMarker = join(fixtureRoot, "tty-observed"); + const runId = basename(fixtureRoot).replaceAll(/[^a-zA-Z0-9]/gu, ""); + const baselinePath = `/tmp/nemoclaw-launch-session-${runId}.json`; + mkdirSync(sessionRoot); try { writeFileSync( - scriptStub, + fakeLaunch, + String.raw`#!/usr/bin/env node +const fs = require("node:fs"); +const readline = require("node:readline"); + +if (!process.stdin.isTTY || !process.stdout.isTTY) process.exit(64); +fs.writeFileSync(process.env.NEMOCLAW_FIXTURE_TTY_MARKER, ""); +const sessionFile = process.env.NEMOCLAW_FIXTURE_SESSION_FILE; +const mode = process.env.NEMOCLAW_FIXTURE_MODE; +const terminalCopy = process.env.NEMOCLAW_FIXTURE_TERMINAL_COPY; +const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: true }); +const ask = () => new Promise((resolve) => rl.question("", resolve)); +const append = (role, content) => fs.appendFileSync( + sessionFile, + JSON.stringify({ message: { content: [{ text: content, type: "text" }], role }, type: "message" }) + "\n", +); + +(async () => { + const first = await ask(); + if (mode === "invalid-order") { + append("assistant", "response before input"); + append("user", first); + } else { + append("user", first); + process.stdout.write(terminalCopy === "ansi" ? "\u001b[2Kignored repaint\r" : "ignored plain copy\n"); + append("assistant", "first response"); + } + + const second = await ask(); + append("user", second); + append("assistant", "second response"); + const exitCommand = await ask(); + rl.close(); + if (exitCommand !== "/exit") process.exit(65); + process.exit(mode.includes("nonzero") ? 23 : 0); +})().catch(() => process.exit(66)); +`, + ); + writeFileSync( + fakeOpenshell, String.raw`#!/usr/bin/env bash set -euo pipefail -capture="" -for argument in "$@"; do - capture="$argument" -done -: >"$capture" -IFS= read -r -d $'\r' _ -printf '%s\n' "$NEMOCLAW_FIXTURE_REPLY" | tee "$capture" -${ - closeAfterReply - ? "" - : String.raw`IFS= read -r -d $'\r' exit_command -[[ "$exit_command" == "/exit" ]]` -} -exit ${exitStatus} +if [[ "$NEMOCLAW_FIXTURE_MODE" == *"cleanup-failure" && " $* " == *" rm -f -- "* ]]; then + exit 71 +fi +while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done +[[ "$#" -gt 0 ]] +shift +exec "$@" `, ); - writeFileSync(sleepStub, '#!/bin/sh\nif [ "${1:-}" = "0.1" ]; then /bin/sleep 0.01; fi\n'); - writeFileSync(timeoutStub, '#!/bin/sh\nshift 2\nexec "$@"\n'); - chmodSync(scriptStub, 0o755); - chmodSync(sleepStub, 0o755); - chmodSync(timeoutStub, 0o755); + chmodSync(fakeLaunch, 0o755); + chmodSync(fakeOpenshell, 0o755); - return spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { + const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { encoding: "utf8", env: { ...process.env, - NEMOCLAW_LAUNCH_COMMAND: "ignored", + NEMOCLAW_FIXTURE_MODE: mode, + NEMOCLAW_FIXTURE_SESSION_FILE: join(sessionRoot, "session-a.jsonl"), + NEMOCLAW_FIXTURE_TERMINAL_COPY: terminalCopy, + NEMOCLAW_FIXTURE_TTY_MARKER: ttyMarker, + NEMOCLAW_LAUNCH_COMMAND: fakeLaunch, NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: closeAfterReply ? "" : "/exit", - NEMOCLAW_LAUNCH_EXPECTED_REPLY: "PONG", - NEMOCLAW_FIXTURE_REPLY: reply, - NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT: "", - NEMOCLAW_LAUNCH_PROMPT: "prompt", - NEMOCLAW_LAUNCH_READY_TEXT: "", + NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", + NEMOCLAW_LAUNCH_FIRST_INPUT: "first input", + NEMOCLAW_LAUNCH_RUN_ID: runId, NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - PATH: `${fixtureRoot}:${process.env.PATH ?? ""}`, + NEMOCLAW_LAUNCH_SECOND_INPUT: "second input", + NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT: OPENCLAW_SESSION_EVIDENCE_SCRIPT, + NEMOCLAW_LAUNCH_SESSION_ROOT: sessionRoot, + NEMOCLAW_OPENSHELL_COMMAND: fakeOpenshell, + TERM: "xterm-256color", }, - timeout: 10_000, + timeout: 15_000, }); + + return { + baselineRemoved: !existsSync(baselinePath), + result, + ttyObserved: existsSync(ttyMarker), + }; } finally { rmSync(fixtureRoot, { force: true, recursive: true }); + rmSync(baselinePath, { force: true }); } } +it("qualifies two ordered structured turns without comparing message content (#9160)", () => { + const { baseline, baselineMode, qualification } = runEvidenceFixture({ + after: { + "session-a": [ + message("user", "first arbitrary input"), + message("assistant", "first arbitrary response"), + message("user", "different second input"), + message("assistant", "different second response"), + ], + }, + expectedTurns: 2, + }); + + expect(baseline.status).toBe(0); + expect(baselineMode).toBe(0o600); + expect(qualification.status).toBe(0); +}); + +it("keeps a partial structured turn pending (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + after: { "session-a": [message("user")] }, + expectedTurns: 1, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(1); +}); + +it("does not qualify structured turns recorded before the baseline (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + before: { "session-a": [message("user"), message("assistant")] }, + after: {}, + expectedTurns: 1, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(1); +}); + +it("rejects malformed, empty, duplicated, extra, out-of-order, or cross-session records (#9160)", () => { + const cases: SessionRecords[] = [ + { "session-a": [message("assistant"), message("user")] }, + { "session-a": [message("user"), message("user"), message("assistant")] }, + { "session-a": [message("user"), message("assistant"), message("assistant")] }, + { "session-a": [message("user"), "not-json", message("assistant")] }, + { "session-a": [emptyMessage("user"), message("assistant")] }, + { "session-a": [message("user"), message("assistant")], "session-b": [message("user")] }, + ]; + + for (const after of cases) { + const { baseline, qualification } = runEvidenceFixture({ after, expectedTurns: 1 }); + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); + } +}); + +it("rejects an unterminated appended session record (#9160)", () => { + const { baseline, qualification } = runEvidenceFixture({ + after: { + "session-a": [ + message("user"), + message("assistant"), + message("user"), + message("assistant"), + message("user"), + ], + }, + afterFinalNewline: false, + expectedTurns: 2, + }); + + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); +}); + +it("rejects an invalid baseline or a removed, rewritten, or truncated session (#9160)", () => { + for (const mutation of ["invalid", "removed", "rewritten", "truncated"] as const) { + const { baseline, qualification } = runBaselineMutationFixture(mutation); + expect(baseline.status).toBe(0); + expect(qualification.status).toBe(2); + } +}); + +it.runIf(process.platform === "linux")( + "sends two inputs and exit through a real PTY without using terminal copy as evidence (#9160)", + () => { + for (const terminalCopy of ["ansi", "plain"] as const) { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "valid", + terminalCopy, + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + } + }, +); + it.runIf(process.platform === "linux")( - "runs producer then two distinct PTY launch turns under one lease (#8942)", + "rejects out-of-order structured records even when the PTY process remains active (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "invalid-order", + "plain", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + }, +); + +it.runIf(process.platform === "linux")( + "propagates a nonzero TUI exit after two structured turns (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture("nonzero", "plain"); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(true); + expect(result.signal).toBeNull(); + expect(result.status).toBe(23); + }, +); + +it.runIf(process.platform === "linux")( + "fails when a successful PTY session cannot remove its structured baseline (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "cleanup-failure", + "plain", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(false); + expect(result.signal).toBeNull(); + expect(result.status).toBe(1); + }, +); + +it.runIf(process.platform === "linux")( + "preserves a nonzero PTY exit when structured baseline cleanup also fails (#9160)", + () => { + const { baselineRemoved, result, ttyObserved } = runLaunchSessionFixture( + "nonzero-cleanup-failure", + "plain", + ); + + expect(ttyObserved).toBe(true); + expect(baselineRemoved).toBe(false); + expect(result.signal).toBeNull(); + expect(result.status).toBe(23); + }, +); + +it.runIf(process.platform === "linux")( + "runs the producer then two PTY launch sessions under one lease (#8942, #9023, #9160)", async () => { const calls: Array<{ command: string; args: string[]; env?: NodeJS.ProcessEnv }> = []; let launchPhaseStartedAtCallCount = -1; const host = { - command: async (command: string, args: string[], options: { env?: NodeJS.ProcessEnv }) => { - calls.push({ command, args, env: options.env }); - return { - exitCode: 0, - signal: null, - stdout: args.includes("--probe-only") ? "Probe complete" : "NEMOCLAW_LAUNCH_TURN_OK", - stderr: "", - }; + command: async (command: string, args: string[], options?: { env?: NodeJS.ProcessEnv }) => { + calls.push({ command, args, env: options?.env }); + return { exitCode: 0, signal: null, stdout: "", stderr: "" }; }, + openshellCommandPath: "openshell", }; - await runLaunchReadinessLeaseTurns({ + await runOpenClawLaunchReadinessLeaseTurns({ artifactName: "lease-turn", cliCommand: "node", cliEntrypoint: "/repo/bin/nemoclaw.js", env: {}, exitCommand: "/exit", host: host as never, - postReplyReadyText: "gateway connected | idle", - readyText: "gateway connected | idle", redactionValues: [], sandboxName: "alpha", beforeLaunchTurns: () => { @@ -97,167 +411,38 @@ it.runIf(process.platform === "linux")( }, }); - expect(calls).toHaveLength(3); expect(launchPhaseStartedAtCallCount).toBe(1); + expect(calls).toHaveLength(3); expect(calls[0]).toMatchObject({ command: "node", args: ["/repo/bin/nemoclaw.js", "alpha", "connect", "--probe-only"], }); - expect(calls[1]?.env?.NEMOCLAW_LAUNCH_EXPECTED_REPLY).not.toBe( - calls[2]?.env?.NEMOCLAW_LAUNCH_EXPECTED_REPLY, - ); + expect(calls.slice(1).map((call) => call.command)).toEqual(["bash", "bash"]); + expect(calls.slice(1).map((call) => call.args)).toEqual([ + ["-lc", LAUNCH_TURN_SCRIPT], + ["-lc", LAUNCH_TURN_SCRIPT], + ]); expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_EXIT_COMMAND)).toEqual([ "/exit", "/exit", ]); - expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_READY_TEXT)).toEqual([ - "gateway connected | idle", - "gateway connected | idle", + expect(calls.slice(1).map((call) => call.env?.NEMOCLAW_OPENSHELL_COMMAND)).toEqual([ + "openshell", + "openshell", ]); - expect( - calls.slice(1).map((call) => call.env?.NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT), - ).toEqual(["gateway connected | idle", "gateway connected | idle"]); - }, -); - -it.runIf(process.platform !== "win32")( - "requires the exact reply before the post-reply 'gateway connected | idle' line (#9023)", - () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "nemoclaw-launch-turn-ready-")); - const scriptStub = join(fixtureRoot, "script"); - const sleepStub = join(fixtureRoot, "sleep"); - const timeoutStub = join(fixtureRoot, "timeout"); - - try { - writeFileSync( - scriptStub, - String.raw`#!/usr/bin/env bash -set -euo pipefail -capture="" -for argument in "$@"; do - capture="$argument" -done -: >"$capture" -if IFS= read -r -t 1 -d $'\r' _; then - echo "prompt arrived before gateway readiness" >&2 - exit 1 -fi -printf 'gateway connected | idle\n' | tee -a "$capture" -IFS= read -r -d $'\r' _ -printf 'gateway connected | idle\n' | tee -a "$capture" -if IFS= read -r -t 1 -d $'\r' _; then - echo "exit command arrived before the exact reply" >&2 - exit 1 -fi -printf 'PONG\n' | tee -a "$capture" -printf 'gateway connected | busy\n' | tee -a "$capture" -if IFS= read -r -t 1 -d $'\r' _; then - echo "exit command arrived while the TUI reported 'gateway connected | busy'" >&2 - exit 1 -fi -printf 'gateway connected | idle\n' | tee -a "$capture" -IFS= read -r -d $'\r' exit_command -[[ "$exit_command" == "/exit" ]] -exit 0 -`, + for (const call of calls.slice(1)) { + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_EXPECTED_REPLY"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_PROMPT"); + expect(call.env).not.toHaveProperty("NEMOCLAW_LAUNCH_READY_TEXT"); + expect(typeof call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).toBe("string"); + expect(typeof call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT).toBe("string"); + expect(call.env?.NEMOCLAW_LAUNCH_FIRST_INPUT).not.toBe( + call.env?.NEMOCLAW_LAUNCH_SECOND_INPUT, + ); + expect(call.env?.NEMOCLAW_LAUNCH_SESSION_EVIDENCE_SCRIPT).toBe( + OPENCLAW_SESSION_EVIDENCE_SCRIPT, ); - writeFileSync(sleepStub, "#!/bin/sh\n/bin/sleep 0.1\n"); - writeFileSync(timeoutStub, '#!/bin/sh\nshift 2\nexec "$@"\n'); - chmodSync(scriptStub, 0o755); - chmodSync(sleepStub, 0o755); - chmodSync(timeoutStub, 0o755); - - const result = spawnSync("bash", ["-c", LAUNCH_TURN_SCRIPT], { - encoding: "utf8", - env: { - ...process.env, - NEMOCLAW_LAUNCH_COMMAND: "ignored", - NEMOCLAW_LAUNCH_ENTRYPOINT: "", - NEMOCLAW_LAUNCH_EXIT_COMMAND: "/exit", - NEMOCLAW_LAUNCH_EXPECTED_REPLY: "PONG", - NEMOCLAW_LAUNCH_PROMPT: "prompt", - NEMOCLAW_LAUNCH_POST_REPLY_READY_TEXT: "gateway connected | idle", - NEMOCLAW_LAUNCH_READY_TEXT: "gateway connected | idle", - NEMOCLAW_LAUNCH_SANDBOX: "sandbox", - PATH: `${fixtureRoot}:${process.env.PATH ?? ""}`, - }, - timeout: 10_000, - }); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("NEMOCLAW_LAUNCH_TURN_OK"); - } finally { - rmSync(fixtureRoot, { force: true, recursive: true }); - } - }, -); - -it.runIf(process.platform !== "win32")( - "records a successful reply and exit status 0 after the TUI exit command (#8584)", - () => { - const result = runLaunchTurnFixture(0); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("NEMOCLAW_LAUNCH_TURN_OK"); - }, -); - -it.runIf(process.platform !== "win32")( - "accepts an exact reply wrapped in a terminated OSC-8 hyperlink (#9023)", - () => { - for (const terminator of ["\u0007", "\u001b\\"]) { - const reply = - `\u001b]8;;https://example.invalid/reply${terminator}` + - `PONG\u001b]8;;${terminator}`; - const result = runLaunchTurnFixture(0, reply); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("NEMOCLAW_LAUNCH_TURN_OK"); - } - }, -); - -it.runIf(process.platform !== "win32")( - "accepts an exact reply after a CSI erase-in-line sequence", - () => { - const result = runLaunchTurnFixture(0, "\u001b[2KPONG"); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status, result.stderr).toBe(0); - expect(result.stdout).toContain("NEMOCLAW_LAUNCH_TURN_OK"); - }, -); - -it.runIf(process.platform !== "win32")( - "rejects a reply token embedded in extra prose (#8942)", - () => { - for (const reply of [ - "The answer is PONG, with extra prose.", - "The answer is \u001b[31mPONG\u001b[0m, with extra prose.", - "\u001b]8;;https://example.invalid/reply\u0007PONG with extra prose\u001b]8;;\u0007", - "\u001b]8;;https://example.invalid/replyPONG", - ]) { - const result = runLaunchTurnFixture(0, reply, true); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status).toBe(1); - expect(result.stderr).toContain("launch did not produce the expected agent reply"); - expect(result.stdout).not.toContain("NEMOCLAW_LAUNCH_TURN_OK"); } }, ); - -it.runIf(process.platform !== "win32")( - "reports a nonzero TUI exit after recording a successful reply (#8584)", - () => { - const result = runLaunchTurnFixture(23); - - expect(result.signal, result.stderr).toBeNull(); - expect(result.status).toBe(23); - expect(result.stderr).toContain("launch exited with status 23"); - expect(result.stdout).not.toContain("NEMOCLAW_LAUNCH_TURN_OK"); - }, -);