Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 175 additions & 1 deletion test/deepagents-code-tui-startup-check.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ const tuiStartupCheckPath = path.join(
const tuiStartupCheckSource = fs
.readFileSync(tuiStartupCheckPath, "utf8")
.replace('\nif [[ "${BASH_SOURCE[0]}" == "$0" ]]; then\n main "$@"\nfi\n', "\n");
const tuiExpectProgram =
tuiStartupCheckSource.match(/expect <<'EXPECT'\n([\s\S]*?)\nEXPECT/)?.[1] ??
(() => {
throw new Error("Deep Agents Code TUI check is missing its embedded Expect program");
})();

function runTuiStartupCheckHelper(snippet: string, env: NodeJS.ProcessEnv = {}): string {
return execFileSync("bash", ["-s"], {
Expand Down Expand Up @@ -48,6 +53,116 @@ function secretFixture(...parts: string[]): string {
return parts.join("");
}

type TuiExpectEvent = "eof" | "exit" | "onboarding" | "ready" | "timeout";

const tclEventLiterals: Record<TuiExpectEvent, string> = {
eof: "{eof}",
exit: "{exit}",
onboarding: "{onboarding}",
ready: "{ready}",
timeout: "{timeout}",
};
const tclshAvailable =
spawnSync("tclsh", ["-"], { encoding: "utf8", input: "exit 0\n" }).status === 0;
const itWithTclsh = it.runIf(tclshAvailable);

function runTuiExpectStateMachine(
events: TuiExpectEvent[],
options: { closeAfterFirstCtrlC?: boolean } = {},
) {
const captureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-dcode-tui-expect-"));
const capture = path.join(captureDir, "raw.log");
const markers = path.join(captureDir, "markers.log");
const trace = path.join(captureDir, "trace.log");
fs.writeFileSync(capture, "");
fs.writeFileSync(markers, "");

const prelude = String.raw`
rename after real_after
rename exit real_exit
set ::fake_events [list ${events.map((event) => tclEventLiterals[event]).join(" ")}]
set ::fake_sent {}
set ::fake_closed 0

proc log_file {args} {}
proc spawn {args} {}
proc after {args} {}
proc send {args} {
binary scan [lindex $args end] H* key_hex
if {$::fake_closed} {
error "fake spawn id is closed"
}
lappend ::fake_sent $key_hex
if {$key_hex eq "03" && $::env(NEMOCLAW_TUI_CLOSE_AFTER_FIRST_CTRL_C) eq "1"} {
set ::fake_closed 1
}
}
proc expect {branches} {
if {[llength $::fake_events] == 0} {
error "fake Expect event queue exhausted"
}
set event [lindex $::fake_events 0]
set ::fake_events [lrange $::fake_events 1 end]
switch -- $event {
onboarding {
set branch_index [lsearch -exact $branches {$onboarding_pattern}]
set ::expect_out(0,string) "Your name (optional)"
}
ready {
set branch_index [lsearch -exact $branches {$ready_pattern}]
set ::expect_out(0,string) "What would you like to build?"
}
exit {
set branch_index [lsearch -glob $branches {NEMOCLAW_TUI_EXIT:*}]
set ::expect_out(0,string) "NEMOCLAW_TUI_EXIT:0"
set ::expect_out(1,string) "0"
}
timeout {
set branch_index [lsearch -exact $branches timeout]
}
eof {
set branch_index [lsearch -exact $branches eof]
}
default {
error "unsupported fake Expect event: $event"
}
}
if {$branch_index < 0} {
error "fake Expect event $event has no matching branch"
}
uplevel 1 [lindex $branches [expr {$branch_index + 1}]]
}
proc exit {{code 0}} {
set trace_file [open $::env(NEMOCLAW_TUI_TRACE) w]
puts $trace_file [join $::fake_sent ,]
close $trace_file
real_exit $code
}
`;
const result = spawnSync("tclsh", ["-"], {
encoding: "utf8",
env: {
...process.env,
NEMOCLAW_TUI_CAPTURE: capture,
NEMOCLAW_TUI_CLOSE_AFTER_FIRST_CTRL_C: options.closeAfterFirstCtrlC ? "1" : "0",
NEMOCLAW_TUI_MARKERS: markers,
NEMOCLAW_TUI_ONBOARDING_PATTERN:
"(your name \\(optional\\)|what should deep agents call you)",
NEMOCLAW_TUI_READY_PATTERN:
"(what would you like|enter (your )?(task|message|prompt)|how can i help)",
NEMOCLAW_TUI_SANDBOX_NAME: "fake-deepagents",
NEMOCLAW_TUI_TIMEOUT: "5",
NEMOCLAW_TUI_TRACE: trace,
},
input: `${prelude}\n${tuiExpectProgram}\n`,
});

const markerText = fs.readFileSync(markers, "utf8");
const traceText = fs.existsSync(trace) ? fs.readFileSync(trace, "utf8").trim() : "";
fs.rmSync(captureDir, { force: true, recursive: true });
return { markerText, result, traceText };
}

describe("Deep Agents Code TUI startup check helpers", () => {
it("rejects unsafe TUI startup timeout values before sandbox execution", () => {
const validate = (timeout: string) =>
Expand Down Expand Up @@ -100,11 +215,69 @@ describe("Deep Agents Code TUI startup check helpers", () => {

expect(readiness("Deep Agents Code starting...\nLoading tools...")).toBe("not-ready");
expect(readiness("Press Enter to continue")).toBe("not-ready");
expect(readiness("Your name (optional)")).toBe("not-ready");
expect(readiness("What should Deep Agents call you?")).toBe("not-ready");
expect(readiness("What would you like to do next?")).toBe("ready");
expect(readiness("Enter your task, then press Enter")).toBe("ready");
expect(readiness("How can I help with the codebase today?")).toBe("ready");
});

it("matches only the pinned first-run onboarding name screen", () => {
const isOnboarding = (capture: string) =>
runTuiStartupCheckHelper(
'if printf "%s" "$CAPTURE" | grep -Eiq "$TUI_ONBOARDING_PATTERN"; then printf onboarding; else printf other; fi',
{ CAPTURE: capture },
);

expect(isOnboarding("Your name (optional)")).toBe("onboarding");
expect(isOnboarding("What should Deep Agents call you?")).toBe("onboarding");
Comment thread
coderabbitai[bot] marked this conversation as resolved.
expect(isOnboarding("Your project name")).toBe("other");
expect(isOnboarding("What would you like to build?")).toBe("other");
});

itWithTclsh("skips first-run onboarding before marking the real TUI prompt ready (tclsh)", () => {
const { markerText, result, traceText } = runTuiExpectStateMachine([
"onboarding",
"ready",
"exit",
]);

expect(result.status, result.stderr).toBe(0);
expect(traceText).toBe("1b,03,03");
expect(markerText).toContain("Your name (optional)");
expect(markerText).toContain("What would you like to build?");
expect(markerText).toContain("NEMOCLAW_TUI_ONBOARDING_SKIPPED");
expect(markerText).toContain("NEMOCLAW_TUI_READY");
expect(markerText.indexOf("NEMOCLAW_TUI_ONBOARDING_SKIPPED")).toBeLessThan(
markerText.indexOf("NEMOCLAW_TUI_READY"),
);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
expect(markerText).toContain("NEMOCLAW_TUI_EXIT_CAPTURED:0");
});

itWithTclsh(
"does not mark the TUI ready when the coding prompt times out after onboarding (tclsh)",
() => {
const { markerText, result, traceText } = runTuiExpectStateMachine(["onboarding", "timeout"]);

expect(result.status, result.stderr).toBe(20);
expect(traceText).toBe("1b,03");
expect(markerText).toContain("NEMOCLAW_TUI_ONBOARDING_SKIPPED");
expect(markerText).toContain("NEMOCLAW_TUI_TIMEOUT");
expect(markerText).not.toContain("NEMOCLAW_TUI_READY");
},
);

itWithTclsh("captures a clean exit when dcode closes after the first Ctrl-C (tclsh)", () => {
const { markerText, result, traceText } = runTuiExpectStateMachine(["ready", "exit"], {
closeAfterFirstCtrlC: true,
});

expect(result.status, result.stderr).toBe(0);
expect(traceText).toBe("03");
expect(markerText).toContain("NEMOCLAW_TUI_READY");
expect(markerText).toContain("NEMOCLAW_TUI_EXIT_CAPTURED:0");
});

it("does not treat generic TUI exit status 1 as a clean Ctrl-C exit", () => {
const assertExit = (exitCode: string) =>
runTuiStartupCheckHelper(
Expand Down Expand Up @@ -139,7 +312,7 @@ describe("Deep Agents Code TUI startup check helpers", () => {
"sandbox_exec() { printf 'NEMOCLAW_DCODE_PROBE:deepagents\\n'; }",
"ensure_expect_available() { return 0; }",
"run_tui_expect() {",
' printf "What would you like to do next?\\nNEMOCLAW_TUI_READY\\nNEMOCLAW_TUI_EXIT_CAPTURED:130\\n" >>"$2"',
' printf "Your name (optional)\\nNEMOCLAW_TUI_ONBOARDING_SKIPPED\\nWhat would you like to do next?\\nNEMOCLAW_TUI_READY\\nNEMOCLAW_TUI_EXIT_CAPTURED:130\\n" >>"$2"',
" return 0",
"}",
"main",
Expand All @@ -152,6 +325,7 @@ describe("Deep Agents Code TUI startup check helpers", () => {
expect(result.stdout).toContain("finite expect harness reached startup and observed exit");
expect(result.stdout).toContain("dcode TUI rendered a usable startup prompt signature");
expect(result.stdout).toContain("dcode TUI exited cleanly after Ctrl-C (exit 130)");
expect(sanitizedText).toContain("NEMOCLAW_TUI_ONBOARDING_SKIPPED");
expect(sanitizedText).toContain("NEMOCLAW_TUI_READY");
expect(sanitizedText).toContain("NEMOCLAW_TUI_EXIT_CAPTURED:130");
} finally {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,11 @@ CONTEXT_SECRET_VALUE_PATTERN='[A-Za-z0-9_.+\/=-]{10,}'
# Upstream dcode does not expose a stable machine-readable TUI ready marker.
# Keep this localized heuristic prompt-shaped; do not match banner-only text.
TUI_READY_PATTERN='(what would you like|what do you want|enter (your )?(task|message|prompt)|describe (the )?(task|change)|how can i help)'
# New dcode homes enter a first-run modal before the coding prompt. Match only
# the pinned upstream name screen so Expect can take its documented skip path.
# deepagents-code 0.1.12 has no non-interactive first-run switch for its name screen.
# Remove this compatibility path once the pinned TUI exposes a stable skip or ready contract.
TUI_ONBOARDING_PATTERN='(your name \(optional\)|what should deep agents call you)'
SENSITIVE_CAPTURE_FILES=()

ok() { printf '%s\n' "${PREFIX}: OK ($*)"; }
Expand Down Expand Up @@ -142,6 +147,7 @@ run_tui_expect() {
env \
NEMOCLAW_TUI_CAPTURE="$raw_capture_file" \
NEMOCLAW_TUI_MARKERS="$marker_capture_file" \
NEMOCLAW_TUI_ONBOARDING_PATTERN="$TUI_ONBOARDING_PATTERN" \
NEMOCLAW_TUI_READY_PATTERN="$TUI_READY_PATTERN" \
NEMOCLAW_TUI_SANDBOX_NAME="$SANDBOX_NAME" \
NEMOCLAW_TUI_TIMEOUT="$TUI_TIMEOUT" \
Expand All @@ -150,6 +156,7 @@ set timeout $env(NEMOCLAW_TUI_TIMEOUT)
set sandbox $env(NEMOCLAW_TUI_SANDBOX_NAME)
set capture $env(NEMOCLAW_TUI_CAPTURE)
set markers $env(NEMOCLAW_TUI_MARKERS)
set onboarding_pattern $env(NEMOCLAW_TUI_ONBOARDING_PATTERN)
set ready_pattern $env(NEMOCLAW_TUI_READY_PATTERN)
log_file -a $capture

Expand All @@ -161,12 +168,18 @@ proc append_marker {markers marker} {

set cmd [list openshell sandbox exec --name $sandbox --tty -- sh -lc {export TERM=xterm-256color; cd /sandbox; dcode; status=$?; printf "\nNEMOCLAW_TUI_EXIT:%s\n" "$status"}]
spawn {*}$cmd
set saw_onboarding 0
set ready_match ""
expect {
-nocase -re $ready_pattern {
set ready_match $expect_out(0,string)
}
-nocase -re $onboarding_pattern {
append_marker $markers "$expect_out(0,string)"
append_marker $markers "NEMOCLAW_TUI_READY"
puts "\nNEMOCLAW_TUI_READY"
send -- "\003"
append_marker $markers "NEMOCLAW_TUI_ONBOARDING_SKIPPED"
puts "\nNEMOCLAW_TUI_ONBOARDING_SKIPPED"
send -- "\033"
set saw_onboarding 1
}
timeout {
append_marker $markers "NEMOCLAW_TUI_TIMEOUT"
Expand All @@ -181,6 +194,33 @@ expect {
}
}

if {$saw_onboarding} {
expect {
-nocase -re $ready_pattern {
set ready_match $expect_out(0,string)
}
timeout {
append_marker $markers "NEMOCLAW_TUI_TIMEOUT"
puts "\nNEMOCLAW_TUI_TIMEOUT"
send -- "\003"
exit 20
}
eof {
append_marker $markers "NEMOCLAW_TUI_EOF_BEFORE_READY"
puts "\nNEMOCLAW_TUI_EOF_BEFORE_READY"
exit 21
}
}
}

append_marker $markers "$ready_match"
append_marker $markers "NEMOCLAW_TUI_READY"
puts "\nNEMOCLAW_TUI_READY"
# Idle dcode arms quit on the first Ctrl-C and exits on the second.
send -- "\003"
after 250
catch {send -- "\003"}

set timeout 20
expect {
-re {NEMOCLAW_TUI_EXIT:([0-9]+)} {
Expand Down
6 changes: 6 additions & 0 deletions test/langchain-deepagents-code-image.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,12 @@ describe("LangChain Deep Agents Code image contracts", () => {
expect(tuiStartupCheck).toContain("unable to probe sandbox");
expect(tuiStartupCheck).toContain("unexpected sandbox probe output");
expect(tuiStartupCheck).toContain("cd /sandbox; dcode");
expect(tuiStartupCheck).toContain('NEMOCLAW_TUI_ONBOARDING_PATTERN="$TUI_ONBOARDING_PATTERN"');
expect(tuiStartupCheck).toContain("-nocase -re $onboarding_pattern");
expect(tuiStartupCheck).toContain('append_marker $markers "NEMOCLAW_TUI_ONBOARDING_SKIPPED"');
expect(tuiStartupCheck).toContain('send -- "\\033"');
expect(tuiStartupCheck).toContain("if {$saw_onboarding}");
expect(tuiStartupCheck).toContain('send -- "\\003"\nafter 250\ncatch {send -- "\\003"}');
expect(tuiStartupCheck).toContain('append_marker $markers "$expect_out(0,string)"');
expect(tuiStartupCheck).toContain('append_marker $markers "NEMOCLAW_TUI_READY"');
expect(tuiStartupCheck).toContain('append_marker $markers "NEMOCLAW_TUI_TIMEOUT"');
Expand Down
Loading