From 0f3b3b2f0ab06f68871b2cc49cd7d3ccae2025a3 Mon Sep 17 00:00:00 2001 From: Thomas Luizon Rodrigues Gregorio Date: Mon, 27 Jul 2026 22:13:01 -0300 Subject: [PATCH] fix: harden worker launch tooling for ORB-126 --- tools/check-ticket.mjs | 4 +- tools/launch-worker.mjs | 36 ++++++++++++- tools/test-tools.mjs | 111 ++++++++++++++++++++++++++++++++++++++-- tools/worker-watch.mjs | 3 +- 4 files changed, 146 insertions(+), 8 deletions(-) diff --git a/tools/check-ticket.mjs b/tools/check-ticket.mjs index 134fc46ab8..8d72a5f9b0 100644 --- a/tools/check-ticket.mjs +++ b/tools/check-ticket.mjs @@ -111,7 +111,9 @@ if (mode === "--file") { validateLabels(labels) const relations = parsedResult.relations ?? issue.relations ?? [] const blockedBy = relations.filter((r) => r.relationship === "blockedBy" || r.type === "blockedBy") - if (/\b(after|once|depends on|blocked by)\b/i.test(body) && blockedBy.length === 0) { + const namesIssueDependency = + /\b(?:after|once|depends on|blocked by)\s+(?:issue\s+)?(?:`|\[)?[A-Z]+-\d+\b/i.test(body) + if (namesIssueDependency && blockedBy.length === 0) { problems.push("body PROSE mentions a dependency but the issue has no blockedBy relation; the DAG is explicit, never inferred from titles (6.2)") } } else { diff --git a/tools/launch-worker.mjs b/tools/launch-worker.mjs index 680374b02a..2025ff3c8b 100644 --- a/tools/launch-worker.mjs +++ b/tools/launch-worker.mjs @@ -120,6 +120,8 @@ const busy = (handle) => isRepainting(orca, handle) */ const MAX_POINTER_SENDS = 3 const POINTER_PAINT_MS = 4000 +const MAX_TERMINAL_CREATE_ATTEMPTS = 3 +const TERMINAL_CREATE_BACKOFF_MS = 1000 /** How many settle windows a painting TUI gets before the launch gives up. A TUI that is still * repainting after all of them is never re-sent to: there is no safe moment, and a queued send * would cut its running turn short. */ @@ -261,6 +263,37 @@ const orca = (args) => { return parsed.result ?? parsed } +const createTerminal = (worktreeSelector, command) => { + const args = ["terminal", "create", "--worktree", worktreeSelector, "--command", command] + for (let attempt = 1; attempt <= MAX_TERMINAL_CREATE_ATTEMPTS; attempt += 1) { + const result = spawnSync(ORCA, [...args, "--json"], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 }) + if (result.error) fail(3, `orca terminal create failed: ${result.error.message}`) + + let parsed + try { + parsed = JSON.parse(result.stdout) + } catch { + fail(3, `orca ${args.join(" ")} returned unparseable output: ${(result.stdout || result.stderr || "").slice(0, 400)}`) + } + + const failureMessage = parsed.error?.message || result.stderr?.trim() || "unknown orca error" + const timedOut = parsed.error?.code === "timeout" || /terminal creation timed out/i.test(failureMessage) + if (parsed.ok !== false && result.status === 0) { + const handle = (parsed.result ?? parsed).terminal?.handle + if (!handle) fail(3, "orca terminal create returned no handle") + return handle + } + if (!timedOut) fail(3, `orca ${args.join(" ")} failed: ${failureMessage}`) + if (attempt === MAX_TERMINAL_CREATE_ATTEMPTS) { + fail(3, `orca ${args.join(" ")} failed after ${attempt} attempts: ${failureMessage}`) + } + + const backoff = TERMINAL_CREATE_BACKOFF_MS * attempt + console.error(`orca terminal create timed out (attempt ${attempt} of ${MAX_TERMINAL_CREATE_ATTEMPTS}); retrying in ${backoff}ms`) + pause(backoff) + } +} + /** * `orca terminal wait` reports "not yet" two different ways, and neither is a tool failure: * a condition it cannot meet in time exits 1 with ok:false and error.code timeout, while a @@ -447,8 +480,7 @@ const actualBranch = git(["-C", worktreePath, "rev-parse", "--abbrev-ref", "HEAD if (actualBranch !== branch) fail(3, `expected the worktree on ${branch}, found ${actualBranch}`) console.error(`starting the ${engineName} TUI: ${command}`) -const terminal = orca(["terminal", "create", "--worktree", worktreeSelector, "--command", command]).terminal?.handle -if (!terminal) fail(3, "orca terminal create returned no handle") +const terminal = createTerminal(worktreeSelector, command) let trustPromptAnswered = false let waitAttempts = 0 diff --git a/tools/test-tools.mjs b/tools/test-tools.mjs index a16bb5c5ad..65dc05a5dc 100644 --- a/tools/test-tools.mjs +++ b/tools/test-tools.mjs @@ -109,7 +109,8 @@ const BASH = resolveBash() */ const ORCA_SHIM = stage( "orca-shim.cjs", - `const { appendFileSync, existsSync, readFileSync, rmSync } = require("node:fs") + `const { spawnSync } = require("node:child_process") +const { appendFileSync, existsSync, readFileSync, rmSync } = require("node:fs") const argv = process.argv.slice(1) if (argv[0] && existsSync(argv[0])) return const line = argv.join(" ") @@ -121,17 +122,21 @@ if (!match) { } if (match.delayMs) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, match.delayMs) if (match.removePath) rmSync(match.removePath, { recursive: true, force: true }) +if (match.pruneRepo) spawnSync("git", ["-C", match.pruneRepo, "worktree", "prune"]) let out = match.stdout +let exit = match.exit ?? 0 if (Array.isArray(match.sequence)) { const log = process.env.ORBIT_ORCA_LOG const previous = log && existsSync(log) ? readFileSync(log, "utf8").split("\\n").filter(Boolean).filter((entry) => JSON.parse(entry).join(" ").includes(match.match)).length : 0 - out = match.sequence[Math.min(previous, match.sequence.length - 1)] + const selected = match.sequence[Math.min(previous, match.sequence.length - 1)] + out = typeof selected === "string" ? selected : selected.stdout + if (typeof selected !== "string") exit = selected.exit ?? exit } if (process.env.ORBIT_ORCA_LOG) appendFileSync(process.env.ORBIT_ORCA_LOG, JSON.stringify(argv) + "\\n") process.stdout.write(out.replaceAll("__NOW__", String(Date.now()))) -process.exit(match.exit ?? 0) +process.exit(exit) `, ) @@ -465,6 +470,73 @@ const pointerDeliveryCases = () => { ) } +const runTerminalCreateLaunch = (label, terminalCreateSequence) => { + const staged = stageLaunchWorker(label, INTERACTIVE_WORKER) + const checkout = join(staged.base, "checkout") + const git = (args) => spawnSync("git", ["-C", staged.repoPath, ...args], { encoding: "utf8" }) + for (const args of [ + ["init", "-q", "--initial-branch=main"], + ["config", "user.email", "gate@orbit.test"], + ["config", "user.name", "Orbit Gate"], + ["commit", "-q", "--allow-empty", "-m", "base"], + ["worktree", "add", "-q", "-b", "thomasluizon/orb-75", checkout], + ]) { + if (git(args).status !== 0) return null + } + + const log = join(staged.base, "orca-calls.log") + const promptFile = stage(`${label}-prompt.md`, "the ticket body verbatim\n") + const painted = `> Read ${promptFile} and execute it in full. That file is your complete work order for ORB-75:` + const plan = [ + ...linearIssueStub(["repo:ui"]), + { match: "worktree create", stdout: JSON.stringify({ ok: true, result: { worktree: { path: checkout, branch: "refs/heads/thomasluizon/orb-75" } } }) }, + { match: "terminal create", sequence: terminalCreateSequence }, + { match: "terminal wait", stdout: JSON.stringify({ ok: true, result: { wait: { satisfied: true } } }) }, + { match: "terminal show", stdout: JSON.stringify({ ok: true, result: { terminal: { lastOutputAt: 1785168487585 } } }) }, + { match: "terminal read", stdout: JSON.stringify({ ok: true, result: { terminal: { tail: [painted] } } }) }, + { match: "terminal send", stdout: JSON.stringify({ ok: true, result: {} }) }, + { match: "terminal switch", stdout: JSON.stringify({ ok: true, result: {} }) }, + { match: "worktree set", stdout: JSON.stringify({ ok: true, result: {} }) }, + { match: "terminal stop", stdout: JSON.stringify({ ok: true, result: {} }) }, + { match: "worktree rm", stdout: JSON.stringify({ ok: true, result: {} }), removePath: checkout, pruneRepo: staged.repoPath }, + ] + const result = run("launch-worker.mjs", ["--issue", "ORB-75", "--prompt-file", promptFile], { + path: staged.path, + env: { ...orcaEnv(plan), ORBIT_ORCA_LOG: log }, + }) + const calls = existsSync(log) ? readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)) : [] + const count = (first, second) => calls.filter((argv) => argv[0].split(/[\\/]/).pop() === first && argv[1] === second).length + const branches = git(["branch", "--list", "feature/orb-75-prove-the-harness-gate", "thomasluizon/orb-75"]).stdout.trim() + return { result, terminalCreates: count("terminal", "create"), worktreeCreates: count("worktree", "create"), checkout, branches } +} + +const terminalCreateRetryCases = () => { + const timeout = { stdout: JSON.stringify({ ok: false, error: { code: "timeout", message: "Terminal creation timed out" } }), exit: 1 } + const success = { stdout: JSON.stringify({ ok: true, result: { terminal: { handle: "t1" } } }), exit: 0 } + const recovered = runTerminalCreateLaunch("terminal-create-recovers", [timeout, success]) + if (!recovered) { + T("launch-worker.mjs: terminal create retries can be staged", false, "could not stage a linked git worktree") + return + } + T( + "launch-worker.mjs: a terminal create timeout retries inside the same worktree and then succeeds", + recovered.result.status === 0 && recovered.terminalCreates === 2 && recovered.worktreeCreates === 1 && /"terminal": "t1"/.test(recovered.result.stdout), + `exit ${recovered.result.status}, terminal creates ${recovered.terminalCreates}, worktree creates ${recovered.worktreeCreates}\n ${recovered.result.stderr.trim().split("\n").slice(-5).join("\n ")}`, + ) + + const exhausted = runTerminalCreateLaunch("terminal-create-exhausted", [timeout, timeout, timeout]) + T( + "launch-worker.mjs: terminal create timeout retries are bounded and preserve the timeout cause", + exhausted.result.status === 3 && exhausted.terminalCreates === 3 && /failed after 3 attempts: Terminal creation timed out/.test(exhausted.result.stderr), + `exit ${exhausted.result.status}, terminal creates ${exhausted.terminalCreates}\n ${exhausted.result.stderr.trim().split("\n").slice(-6).join("\n ")}`, + ) + T( + "launch-worker.mjs: exhausting terminal create retries rolls back the one worktree and both branches", + exhausted.worktreeCreates === 1 && !existsSync(exhausted.checkout) && exhausted.branches === "", + `worktree creates ${exhausted.worktreeCreates}, checkout exists ${existsSync(exhausted.checkout)}, branches ${JSON.stringify(exhausted.branches)}`, + ) +} + const launchWorkerCases = () => { const promptFile = stage("prompt.md", "the ticket body verbatim\n") @@ -530,6 +602,7 @@ const launchWorkerCases = () => { trustScreenCases() pointerDeliveryCases() + terminalCreateRetryCases() const launcherSource = readFileSync(join(TOOLS_DIR, "launch-worker.mjs"), "utf8") for (const [clause, pattern] of Object.entries(REQUIRED_CONTRACT_CLAUSES)) { @@ -691,6 +764,7 @@ const prWatchCases = () => { * concatenated repaint fragments that hide whatever it last really said. */ const workerWatchCases = () => { + const terminalHandle = "term_ca852374-175d-42cd-8407-b579a03cc13a" const childWorktree = (path) => ({ path, repoId: "r-ui", @@ -710,7 +784,7 @@ const workerWatchCases = () => { { match: "worktree list", stdout: JSON.stringify({ ok: true, result: { worktrees: [childWorktree(path)] } }) }, { match: "terminal list", - stdout: JSON.stringify({ ok: true, result: { terminals: [{ handle: "term_abc123def4567", worktreePath: path, title: "Claude Code", lastOutputAt: 0 }] } }).replace( + stdout: JSON.stringify({ ok: true, result: { terminals: [{ handle: terminalHandle, worktreePath: path, title: "Claude Code", lastOutputAt: 0 }] } }).replace( '"lastOutputAt":0', `"lastOutputAt":${lastOutputAt}`, ), @@ -744,6 +818,11 @@ const workerWatchCases = () => { ) T("worker-watch.mjs: the repaint tail is stripped to nothing rather than printed raw", /nothing but repaint noise/.test(busy.stdout), busy.stdout.slice(0, 400)) T("worker-watch.mjs: the ticket's Linear state is reported alongside liveness", /In Progress/.test(busy.stdout), busy.stdout.slice(0, 400)) + T( + "worker-watch.mjs: the rendered terminal handle is complete and directly reusable", + busy.stdout.includes(`${terminalHandle} BUSY`) && !/term_ca852374\s+BUSY/.test(busy.stdout), + busy.stdout.slice(0, 400), + ) /** A frozen lastOutputAt is a settled TUI: identical samples, so IDLE. */ const idle = check( @@ -1212,6 +1291,30 @@ const gateCases = { "check-ticket.mjs": () => { check("check-ticket.mjs", "an incomplete body is rejected", ["--file", stage("ticket.md", "# A ticket\n\nno template sections here\n")], { nonZero: true }) check("check-ticket.mjs", "a missing body file is a usage error", ["--file", join(root, "absent.md")], { status: 2 }) + const issue = (sentence) => ({ + match: "linear issue ORB-99", + stdout: JSON.stringify({ + ok: true, + result: { + issue: { identifier: "ORB-99", title: "Keep explicit issue dependencies precise", description: `${VALID_TICKET_BODY}\n\n${sentence}`, labels: [{ name: "repo:api" }] }, + relations: [], + }, + }), + }) + for (const sentence of [ + "Cleanup runs after the terminal exits.", + "The launcher retries once the daemon is responsive.", + "The branch name depends on configuration, and startup can be blocked by a trust prompt.", + ]) { + check("check-ticket.mjs", `ordinary prose does not imply a dependency: ${sentence}`, ["--issue", "ORB-99"], { status: 0, stdout: /ticket ok/ }, { env: orcaEnv([issue(sentence)]) }) + } + check( + "check-ticket.mjs", + "a named issue blocker still requires a blockedBy relation", + ["--issue", "ORB-99"], + { status: 1, stderr: /body PROSE mentions a dependency/ }, + { env: orcaEnv([issue("This change is blocked by ORB-1.")]) }, + ) }, "check-push-target.mjs": () => { check("check-push-target.mjs", "a push to main is blocked", [], { status: 1, stderr: /BLOCKED/ }, { input: "refs/heads/main abc refs/heads/main def\n" }) diff --git a/tools/worker-watch.mjs b/tools/worker-watch.mjs index 067fb2d156..592bcce7cd 100644 --- a/tools/worker-watch.mjs +++ b/tools/worker-watch.mjs @@ -219,7 +219,8 @@ for (const entry of report) { console.log(` path ${entry.path}`) if (entry.linearState) console.log(` linear ${entry.linearState}`) if (entry.comment) console.log(` card ${entry.comment}`) - console.log(` terminals ${entry.terminals.length === 0 ? "none (no TUI attached)" : entry.terminals.map((terminal) => `${terminal.handle.slice(0, 13)} ${terminal.liveness}`).join(", ")}`) + const terminalLines = entry.terminals.map((terminal) => `${terminal.handle} ${terminal.liveness}`).join("\n ") + console.log(` terminals ${terminalLines || "none (no TUI attached)"}`) if (entry.contract) { const line = entry.contract.state === "met"