diff --git a/.agents/skills/harness-adapters/references/harness/omp.md b/.agents/skills/harness-adapters/references/harness/omp.md index 038b8d01829..cb5a9a8c8a4 100644 --- a/.agents/skills/harness-adapters/references/harness/omp.md +++ b/.agents/skills/harness-adapters/references/harness/omp.md @@ -52,6 +52,6 @@ The same file ports the `tool_call` seatbelts and delivers the session-start dig omp has no asynchronous Stop-hook equivalent, so the Claude auto-arm model does not apply; `fm_supervision_model` classifies omp as `extension`, and `fm_omp_extension_owns_supervision` in `../../../bin/fm-wake-lib.sh` is the ownership proof that tolerates the extension's own watcher hand-off. The Pi supervision branch does not run on omp; without the supervision host every actionable wake is delivered to main, and in a home with `config/supervision-host` and no `config/supervision-host-off` the watch extension spawns the host instead of the arm, with Claude's print mode as its headless engine ([`supervision-host.md`](../../../docs/supervision-host.md)). Launch a primary with plain `omp` inside the home (`FM_OMP_HARNESS=omp omp` when starting from a Claude pane); `../../../bin/fm-session-start.sh` prints `OMP_WATCH_EXTENSION: not loaded` when the running session has not loaded both tracked supervision extensions. -[Watcher continuity](../../../docs/watcher-continuity.md#omp-idle-wake-delivery) owns idle wake delivery, restored-wake recovery, editor normalization, draft preservation, and known limits. +[Watcher continuity](../../../docs/watcher-continuity.md#omp-idle-wake-delivery) owns idle wake delivery, restored-wake recovery, stranded wake text, editor normalization, draft preservation, and known limits. `FM_OMP_LIVE_E2E=1 ../../../tests/fm-omp-primary-live-e2e.test.sh` is the opt-in live guard; `../../../tests/fm-omp-harness.test.sh` is the portable regression. A secondmate registered with `remote=1` in `data/secondmates.md`, spawned through the ordinary `../../../bin/fm-spawn.sh --secondmate` path, is refused on omp until a remote host verifies it, as is `../../../bin/fm-remote-secondmate-control.sh launch`; there is no `--remote` flag. diff --git a/.omp/extensions/fm-primary-omp-watch.ts b/.omp/extensions/fm-primary-omp-watch.ts index fbb6f3a5007..e42da436855 100644 --- a/.omp/extensions/fm-primary-omp-watch.ts +++ b/.omp/extensions/fm-primary-omp-watch.ts @@ -79,6 +79,9 @@ // Restored-wake recovery is documented in docs/watcher-continuity.md. // Recovery must use the real editor and resend only this extension's unchanged // emitted wake, never submit the whole composer or alter operator draft bytes. +// Complete wake text left unsent in an idle composer by other wiring is found +// by a low-rate poll and delivered the same way; docs/watcher-continuity.md +// (omp stranded wake text) owns what it may touch. import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; @@ -150,6 +153,8 @@ type SessionGeneration = { // replacement began still needs to distinguish unconsumed wakes from // consumed ones. unconsumedWakes: Map; + restorePendingWaits: number; + queueStuckReported: boolean; // A verified successor's failure close that arrived while the pipeline was // still delivering the wake it was started for; its bounded retry runs once // that delivery settles instead of being skipped by the single-flight guard. @@ -513,6 +518,8 @@ function createGeneration(): SessionGeneration { pendingActionables: [], cleanupFailure: "", unconsumedWakes: new Map(), + restorePendingWaits: 0, + queueStuckReported: false, deferredClose: null, }; } @@ -598,6 +605,65 @@ async function stopSessionGeneration( } } +// The wake body is `` inside the watcher envelope the +// operational-input owner encodes; sendWake builds it and the stranded-wake +// finder below recognizes it, so the two share these parts. +const wakeBodyPrefix = "FIRSTMATE WATCHER WAKE: "; +const wakeBodySuffix = "\n\nRun bin/fm-wake-drain.sh first and handle the queued wake. Watcher continuity is extension-owned."; + +// The envelope header (transport mark included) comes from the owner's encoder, +// never from a literal here. A failed encode answers null and is retried next time. +let watcherEnvelopeHeaderCache: string | null = null; +function watcherEnvelopeHeader(): string | null { + if (watcherEnvelopeHeaderCache !== null) return watcherEnvelopeHeaderCache; + try { + const encoded = encodeFirstmateOperationalInput("watcher", wakeBodyPrefix); + if (!encoded.endsWith(wakeBodyPrefix)) return null; + watcherEnvelopeHeaderCache = encoded.slice(0, -wakeBodyPrefix.length); + } catch { + return null; + } + return watcherEnvelopeHeaderCache; +} + +type StrandedWake = { + // The complete wake as this extension emits it, transport mark restored. + content: string; + // The editor without the wake and one transport blank-line separator. + remainder: string; +}; + +function findStrandedWake(editor: string, eligible: (content: string) => boolean): StrandedWake | null { + if (!editor.includes(wakeBodyPrefix)) return null; + const header = watcherEnvelopeHeader(); + if (!header) return null; + const mark = header.startsWith("\u2063") ? "\u2063" : ""; + const opening = header.slice(mark.length) + wakeBodyPrefix; + let from = 0; + for (;;) { + const at = editor.indexOf(opening, from); + if (at < 0) return null; + from = at + opening.length; + const start = mark && at > 0 && editor[at - 1] === mark ? at - 1 : at; + if (start !== 0 && (start < 2 || editor.slice(start - 2, start) !== "\n\n")) continue; + const closing = editor.indexOf(wakeBodySuffix, from); + if (closing <= from) continue; + if (editor.slice(from, closing).includes(opening)) continue; + const stop = closing + wakeBodySuffix.length; + if (stop !== editor.length && editor.slice(stop, stop + 2) !== "\n\n") continue; + const content = mark + editor.slice(at, stop); + if (!eligible(content)) continue; + let cutStart = start; + let cutEnd = stop; + if (cutStart >= 2 && editor.slice(cutStart - 2, cutStart) === "\n\n") cutStart -= 2; + else if (editor.slice(cutEnd, cutEnd + 2) === "\n\n") cutEnd += 2; + return { + content, + remainder: editor.slice(0, cutStart) + editor.slice(cutEnd), + }; + } +} + function removeRestoredWake(editor: string, content: string): string | null { const marked = content.startsWith("\u2063"); const needle = (marked ? content.slice(1) : content).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); @@ -728,13 +794,19 @@ export default function (pi: ExtensionAPI) { if (!generationIsLive(owner)) return false; const content = encodeFirstmateOperationalInput( "watcher", - `FIRSTMATE WATCHER WAKE: ${message}\n\nRun bin/fm-wake-drain.sh first and handle the queued wake. Watcher continuity is extension-owned.`, + `${wakeBodyPrefix}${message}${wakeBodySuffix}`, ).replace(/\r\n?/g, "\n").replaceAll("\t", " ").replace(/[\x00-\x09\x0b-\x1f]/g, ""); const token = pending?.token ?? `${process.pid}-${Date.now()}-${++replacementCoordinator.nextTokenId}`; owner.unconsumedWakes.set(token, { content, pending }); try { - if (sessionIsIdle()) await pi.sendUserMessage(content); - else await pi.sendUserMessage(content, { deliverAs: "followUp" }); + if (sessionIsIdle()) { + await pi.sendUserMessage(content); + } else { + await pi.sendUserMessage(content, { deliverAs: "followUp" }); + // A follow-up omp never starts a turn for (an idle state read wrongly, + // or the turn ending around the call) raises no agent_end to look again. + scheduleRestoredWakeCheck(owner); + } } catch (error) { owner.unconsumedWakes.delete(token); throw error; @@ -767,8 +839,16 @@ export default function (pi: ExtensionAPI) { // stale with a replaced session throws on use, which only skips the check. const restoreCheckMs = 2000; const restoreAttemptLimit = 3; + // Further checks that wait for omp's queue to drain into a run before an idle + // session's queue counts as stuck; a compaction can hold the queue for a while. + const restorePendingWaitLimit = 15; + const strandedPollMs = 3000; const restoreAttempts = new Map(); + const strandedAttempts = new Map(); let restoreTimer: ReturnType | null = null; + let strandedTimer: ReturnType | null = null; + let strandedSeen = ""; + let nextStrandedId = 0; function rememberContext(ctx: unknown): void { if (typeof ctx === "object" && ctx !== null) latestContext = ctx; @@ -783,14 +863,66 @@ export default function (pi: ExtensionAPI) { } } - function recoverRestoredWake(owner: SessionGeneration): void { - if (!generationIsLive(owner) || owner.unconsumedWakes.size === 0) return; + function editorContext(): any { const ctx = latestContext; - if (!ctx?.hasUI || typeof ctx.isIdle !== "function" || typeof ctx.ui?.getEditorText !== "function" || typeof ctx.ui?.setEditorText !== "function") return; + if (!ctx?.hasUI || typeof ctx.isIdle !== "function" || typeof ctx.ui?.getEditorText !== "function" || typeof ctx.ui?.setEditorText !== "function") return null; + return ctx; + } + + // A wait this extension cannot resolve is shown in omp's own UI, not dropped. + function reportWait(ctx: any, message: string): void { + try { + ctx.ui?.notify?.(message, "warning"); + } catch {} + } + + // omp is idle with messages still queued after the bounded wait, so nothing + // will run them: the follow-up gate refused to start a turn for them. Sending + // an unconsumed wake through the prompt-starting API starts that turn and + // flushes the queue behind it, and the wake's own consumption proves delivery. + // The wake can then reach main twice, which the idempotent drain absorbs. + function flushStuckQueue(owner: SessionGeneration, ctx: any): void { + for (const [token, wake] of [...owner.unconsumedWakes]) { + const attempts = restoreAttempts.get(token) ?? 0; + if (attempts >= restoreAttemptLimit) continue; + restoreAttempts.set(token, attempts + 1); + pi.sendUserMessage(wake.content); + scheduleRestoredWakeCheck(owner); + return; + } + if (owner.queueStuckReported) return; + owner.queueStuckReported = true; + reportWait( + ctx, + `watcher: wake not delivered - omp is idle with queued messages it will not run, and ${restoreAttemptLimit} resubmissions did not start a turn; the wake stays in the durable queue for bin/fm-wake-drain.sh`, + ); + } + + function recoverRestoredWake(owner: SessionGeneration): void { + if (!generationIsLive(owner)) return; + const ctx = editorContext(); + if (!ctx) return; try { - // A turn that already started, or a queue that still holds messages, - // will consume the wake itself. - if (!ctx.isIdle() || ctx.hasPendingMessages?.()) return; + // A turn that already started will consume the wake itself. + if (!ctx.isIdle()) { + owner.restorePendingWaits = 0; + owner.queueStuckReported = false; + return; + } + // Queued messages normally drain into the next run within moments, so the + // check waits for that. Skipping for good would strand a queue omp will + // never run; after the bounded wait the stuck queue is handled below. + const queued = ctx.hasPendingMessages?.() === true; + if (!queued) { + owner.restorePendingWaits = 0; + owner.queueStuckReported = false; + } + if (owner.unconsumedWakes.size === 0) return; + if (queued && owner.restorePendingWaits < restorePendingWaitLimit) { + owner.restorePendingWaits += 1; + scheduleRestoredWakeCheck(owner); + return; + } let editor = String(ctx.ui.getEditorText() ?? ""); for (const [token, wake] of [...owner.unconsumedWakes]) { const attempts = restoreAttempts.get(token) ?? 0; @@ -805,11 +937,20 @@ export default function (pi: ExtensionAPI) { pi.sendUserMessage(wake.content); return; } + if (queued) flushStuckQueue(owner, ctx); } catch {} } - function scheduleRestoredWakeCheck(owner: SessionGeneration): void { - if (restoreTimer || owner.unconsumedWakes.size === 0) return; + // Every later check keeps a pending one. The check that follows an agent_end + // replaces it instead, so the editor is read two full seconds after the run + // ended - the settle time that tells a wake omp drained from one it left behind. + function scheduleRestoredWakeCheck(owner: SessionGeneration, afterRunEnd = false): void { + if (owner.unconsumedWakes.size === 0) return; + if (restoreTimer) { + if (!afterRunEnd) return; + clearTimeout(restoreTimer); + restoreTimer = null; + } const timer = setTimeout(() => { restoreTimer = null; recoverRestoredWake(owner); @@ -818,6 +959,79 @@ export default function (pi: ExtensionAPI) { restoreTimer = timer; } + // Tracked wakes retain ownership even when their composer copies are missing + // or edited; template matching must not bypass exact restored recovery. + // docs/watcher-continuity.md#omp-stranded-wake-text owns polling behavior. + function pollStrandedWake(owner: SessionGeneration): void { + if (!generationIsLive(owner) || lockOwnership() !== "owned") return; + const ctx = editorContext(); + if (!ctx) return; + try { + if (ctx.isIdle() !== true) { + owner.restorePendingWaits = 0; + owner.queueStuckReported = false; + strandedSeen = ""; + return; + } + if (ctx.hasPendingMessages?.() !== true) { + owner.restorePendingWaits = 0; + owner.queueStuckReported = false; + } + const editor = String(ctx.ui.getEditorText() ?? ""); + const tracked = [...owner.unconsumedWakes.values()]; + let unownedEditor = editor; + for (const wake of tracked) { + const remainder = removeRestoredWake(unownedEditor, wake.content); + if (remainder === null) { + strandedSeen = ""; + return; + } + unownedEditor = remainder; + } + const bare = (text: string): string => (text.startsWith("\u2063") ? text.slice(1) : text); + const found = findStrandedWake(editor, (content) => + !tracked.some((wake) => bare(wake.content) === bare(content)) && + (strandedAttempts.get(content) ?? 0) < restoreAttemptLimit, + ); + if (!found) { + strandedSeen = ""; + return; + } + if (strandedSeen !== editor) { + strandedSeen = editor; + return; + } + const attempts = strandedAttempts.get(found.content) ?? 0; + strandedAttempts.set(found.content, attempts + 1); + const token = `stranded-${process.pid}-${++nextStrandedId}`; + owner.unconsumedWakes.set(token, { content: found.content }); + try { + // Idle, so this starts the turn that consumes the wake. The composer is + // cleared only once omp has taken it. + pi.sendUserMessage(found.content); + } catch (error) { + owner.unconsumedWakes.delete(token); + throw error; + } + ctx.ui.setEditorText(found.remainder); + strandedSeen = ""; + scheduleRestoredWakeCheck(owner); + } catch {} + } + + function startStrandedPoll(): void { + if (strandedTimer) return; + const timer = setInterval(() => pollStrandedWake(generation), strandedPollMs); + timer.unref(); + strandedTimer = timer; + } + + function stopStrandedPoll(): void { + if (strandedTimer) clearInterval(strandedTimer); + strandedTimer = null; + strandedSeen = ""; + } + function confirmHandlingDelivery(recovery: { generation: string; watcherPid: string }): { ok: boolean; detail: string; @@ -1305,6 +1519,7 @@ export default function (pi: ExtensionAPI) { if (!generationIsLive(owner)) return { ok: false, message: shuttingDownMessage }; if (lockOwnership() !== "owned") return startArm(owner); replacementCoordinator.receiver = receiveReplacementActionable; + startStrandedPoll(); let pending: PendingActionableClose[] = []; let loadFailure = ""; try { @@ -1343,6 +1558,8 @@ export default function (pi: ExtensionAPI) { rememberContext(ctx); const message = (event as { message?: { role?: unknown; content?: unknown } })?.message; if (!message || message.role !== "user") return; + generation.restorePendingWaits = 0; + generation.queueStuckReported = false; consumeWake(generation, userMessageText(message.content)); }); // A run that ends with a wake still unconsumed either drains it into the next @@ -1350,7 +1567,9 @@ export default function (pi: ExtensionAPI) { pi.on?.("agent_end", (_event, ctx) => { if (!instance.isCurrent()) return; rememberContext(ctx); - scheduleRestoredWakeCheck(generation); + generation.restorePendingWaits = 0; + generation.queueStuckReported = false; + scheduleRestoredWakeCheck(generation, true); }); pi.on?.("session_start", async (_event, ctx) => { @@ -1379,6 +1598,7 @@ export default function (pi: ExtensionAPI) { if (replacementCoordinator.receiver === receiveReplacementActionable) replacementCoordinator.receiver = null; if (restoreTimer) clearTimeout(restoreTimer); restoreTimer = null; + stopStrandedPoll(); rememberContext(ctx); const stopped = generation; recoveryPending = true; @@ -1421,6 +1641,7 @@ export default function (pi: ExtensionAPI) { activate: armFromSession, retire: () => { clearHealTimer(); + stopStrandedPoll(); lifecycle("instance-retired", { generation: generation.id, by: instance.current()?.id }); if (replacementCoordinator.receiver === receiveReplacementActionable) replacementCoordinator.receiver = null; if (restoreTimer) clearTimeout(restoreTimer); diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index eaa1644ddeb..495abad4826 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -1316,14 +1316,13 @@ Three behaviors of omp left Firstmate-injected text unsubmitted or unseen in a l `tests/fm-omp-harness.test.sh` carries restored-wake editor recovery, descendant sessions, and marker self-repair; `tests/fm-composer-lib.test.sh` carries working-box fixtures captured during this verification; backend submit regressions carry dropped-Enter and stale-frame reread protections, and `tests/fm-wake-queue.test.sh` carries the ordinary stalled-loop alarm and idle-ring boundaries. The live guard that refreshes this entry submits real prompts and stays opt-in. -Both recovery probes require the fresh queued wake to be observed in a pending composer after Escape before queue drainage or draft preservation can count as recovery proof. -If restoration is not observed, the probe retries from a fresh busy turn at most three times, then fails rather than accepting ordinary queued delivery as recovery. +Current queue-readiness and post-Escape restoration assertions are described in [omp stranded wake text and the queue panel](#2026-10-08-omp-stranded-wake-text-and-the-queue-panel). ```sh FM_OMP_WAKE_RESTORE_LIVE=1 tests/fm-omp-wake-restore-live-e2e.test.sh ``` -The latest guard run stopped before restored-wake recovery was exercised; see [omp idle wake behind an advisor note](#2026-10-08-omp-idle-wake-behind-an-advisor-note). +The latest complete guard run, on omp 18.8.1, is recorded in [omp stranded wake text and the queue panel](#2026-10-08-omp-stranded-wake-text-and-the-queue-panel). Historical output below predates its fresh queued-wake and post-Escape restoration assertions, pre-parent-repair marker check, successful child-command evidence, and current busy-state checks at observations and submission. These results do not establish the current recovery contract; output for the removed parent Enter recovery is omitted. @@ -1365,12 +1364,64 @@ The same step run against the extension that queued every wake as a follow-up fa not ok - omp (omp/18.8.1) on herdr 0.9.1: a wake reaching an idle lane behind an advisor note did not start a turn ``` -On omp 18.8.1 the guard's next step, the restored-wake probe, fails before recovery is exercised, identically with and without this change, so it does not yet refresh the [omp injected text](#2026-10-06-omp-injected-text-through-herdr) entry: +The guard's later steps on omp 18.8.1 are recorded in [omp stranded wake text and the queue panel](#2026-10-08-omp-stranded-wake-text-and-the-queue-panel). + +### 2026-10-08 omp stranded wake text and the queue panel + +Verified on 2026-10-08 on macOS arm64 against omp 18.8.1 in isolated Herdr 0.9.1 lab sessions. +The idle-session steps use a scripted local OpenAI-compatible model and spend no tokens; the Escape steps use the real model. + +- **Wake text typed into an idle composer is never submitted by omp.** + A complete Firstmate watcher wake pasted into an idle session's composer without Enter stayed there for the whole 22 seconds observed, and no request reached the model, with the extension before this change. + Nothing raises an event when text lands in an idle composer, so the extension has to look for it; its stranded-wake poll delivered the same text as its own turn and cleared only that wake. +- **omp does not redraw its follow-up queue panel for a follow-up an extension queues.** + A wake queued with `sendUserMessage(text, { deliverAs: "followUp" })` behind a running tool call was absent from the `After yield` panel for the 26 seconds observed, yet Escape then restored it to the composer, so it was queued all along, and the extension context read `hasPendingMessages()` true. + omp 18.8.1 refreshes the panel only from its input-submit, dequeue, compaction, and user `message_start` handlers, and the observation above shows none of them ran for the extension's queued follow-up. + Its `queue_update` snapshot event was not delivered to an extension handler registered for it in the same lab. + The guard's earlier panel parse could therefore not see the queued wake unless something else had redrawn the panel. +- **Escape restores every queued follow-up, and recovery takes one wake per run end.** + The lane also queues its own outcome checks, so Escape can restore a signal wake and a `check:` wake together; the extension submits the first and the second returns after the next `agent_end`. + A restored wake stays in the editor for about two seconds, while one rendered composer read through the pane took one to five seconds on this host, so the guard records omp's editor text through a lab-only probe extension and retains the pane reader as a second restoration signal. + The restoration assertion accepts any watcher wake observed after Escape, not only the newly generated signal; the draft probe requires the draft in the same editor text. + If restoration is not observed, the probe retries from a fresh busy turn at most three times, then fails rather than accepting ordinary queued delivery as recovery. + Queue readiness requires the probe's fresh `hasPendingMessages()` answer, bracketed by busy-state and durable signal-row checks; the queue panel is not an acceptance signal, and screen captures remain available for failure diagnostics. + +[Watcher continuity](../watcher-continuity.md#omp-stranded-wake-text) owns the delivery contract, and [restored-wake recovery](../watcher-continuity.md#omp-restored-wake-recovery) owns the wait on queued messages. +The guard exercises the extension under review on the installed omp: + +```sh +FM_OMP_WAKE_RESTORE_LIVE=1 tests/fm-omp-wake-restore-live-e2e.test.sh +``` + +The stranded-wake step now reads the full editor through `fm_backend_herdr_composer_content` and requires the remainder to equal `operator draft beside the wake` exactly; the model log separately checks that the draft was not submitted. +The idle-session launch uses the same box overlay as the spawned lane so the full-editor reader can identify the composer independently of omp's default shape. +The guard also waits for the probe's fresh idle-context answer before typing setup prompts; the module-loaded marker and native idle report can appear before omp accepts input. +The recorded run below predates this stronger assertion: its bottom-row-only check could miss multiline wake text remaining above the draft. ```text -not ok - omp (omp/18.8.1) on herdr 0.9.1: wakelab1 was not submitted into the running turn's follow-up queue (queue rows: 3) +ok - live omp idle wake: omp (omp/18.8.1) on herdr 0.9.1 started its own turn for a wake that reached an idle lane behind an advisor note and left the operator draft unsent +ok - live omp stranded wake: omp (omp/18.8.1) on herdr 0.9.1 delivered wake text an older wiring left unsent in an idle composer as its own turn, cleared only that wake, and left the operator draft as typed +ok - live omp wake restore: omp (omp/18.8.1) on herdr 0.9.1 re-submitted a wake that Esc restored to the composer, and the lane handled it +ok - live omp wake restore: omp (omp/18.8.1) on herdr 0.9.1 left the operator's draft exactly as typed while it re-submitted the wake +ok - live omp markers: omp (omp/18.8.1) on herdr 0.9.1 kept both loaded markers on the session pid 45643 before parent repair and after the parent resumed +ok - live omp busy composer: omp (omp/18.8.1) on herdr 0.9.1 reads empty and pending while a turn runs +ok - live omp busy composer: omp (omp/18.8.1) on herdr 0.9.1 took an injected doorbell mid-turn and left the composer empty +``` + +The strengthened guard passed every step on 2026-10-09 against omp 18.8.1 on Herdr 0.9.3, with the native Herdr integration installed in a disposable agent home and a scripted local model executing the real tools and drain acknowledgements. + +The earlier guard run with the extension as it stood before this change stopped at the stranded-wake step: + +```text +ok - live omp idle wake: omp (omp/18.8.1) on herdr 0.9.1 started its own turn for a wake that reached an idle lane behind an advisor note and left the operator draft unsent +not ok - omp (omp/18.8.1) on herdr 0.9.1: wake text left unsent in an idle composer was never delivered ``` +The guard has no live step for the wait on a queue that never drains; `tests/fm-omp-harness.test.sh` covers it with a fake omp API. +Additional isolated live validation on 2026-10-09, with omp 18.8.1 on Herdr 0.9.3 and a scripted local model, reproduced this state through real omp APIs: Escape restored a tracked wake beside a draft, a non-turn-triggering custom message supplied the context tail, and an explicit follow-up remained queued while omp reported idle. +After the bounded wait, the extension submitted the wake alone, omp consumed the queued follow-up, and the editor retained exactly the unsent draft. +The before-change extension also reproduced the stranded older-wiring wake: it remained above the draft until the live check timed out; the changed extension delivered it and left exactly the draft. + ## Steering-inbox doorbell The steering channel's one behavioral assumption - a real worker agent follows the constant self-describing doorbell line (list the inbox, read and act on its records in numeric order, then `mv` each into `handled/`) - was verified on 2026-08-23 against every installed verified harness, on tmux 3.6a, macOS arm64, on an isolated private socket, driving the REAL `bin/fm-send.sh` end to end (durable record plus doorbell, with one mid-wait re-ring playing the watcher's role). diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 824683dafec..d1b19ae868d 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -95,18 +95,50 @@ Both suites use `tests/watch-lifecycle-expiry.mjs` for shutdown child-close, suc omp restores queued user follow-ups to the composer when a run is interrupted with Escape or a message is dequeued with Alt+Up, so accepting a wake as a follow-up does not prove a turn consumed it. Before recording or sending a wake, `.omp/extensions/fm-primary-omp-watch.ts` normalizes CRLF and CR to LF, expands each tab to three spaces, and strips other C0 controls to match omp's editor restoration. Consumption still matches the emitted text exactly. -Only an accepted user `message_start` carrying the exact emitted text consumes one pending token. `before_agent_start` records context and the loaded build but does not consume a wake: preparation can still be cancelled by Escape. A second identical wake therefore remains recoverable and eligible for replacement handoff, and shutdown during cancelled preparation retains the pending record until the replacement accepts its user message. -While a wake remains unconsumed, `agent_end` schedules one editor check after two seconds; this is not continuous polling. -The check requires the current generation to be live, a UI editor, positive idle state, and no pending messages. +Only an accepted user `message_start` carrying the exact emitted text consumes one pending token. +`before_agent_start` records context and the loaded build but does not consume a wake: preparation can still be cancelled by Escape. +A second identical wake therefore remains recoverable and eligible for replacement handoff, and shutdown during cancelled preparation retains the pending record until the replacement accepts its user message. +While a wake remains unconsumed, `agent_end` schedules one editor check after two seconds, replacing a check already pending so the editor is read two full seconds after the run ended. +Omp accepting the wake as a follow-up schedules one too, unless a check is already pending. +The check requires the current generation to be live, a UI editor, and positive idle state. +While omp still reports queued messages the check waits for them to drain into a run, in up to fifteen further two-second rounds, rather than skipping for good. +The wait count and once-per-episode warning belong to the session generation. +A replacement starts with both cleared; accepted user `message_start` and `agent_end` clear both even when a whole turn runs between timer checks. +Checks that observe a running turn or an empty queue also clear both for the next episode, including when no wakes remain tracked. Recovery accepts only a complete unchanged emitted wake segment bounded by editor edges or omp's blank-line joins, with only its leading invisible transport mark allowed to be present or absent. -Direct prefix, suffix, or internal edits are left untouched and not submitted. +Direct prefix, suffix, or internal edits are left untouched; edited composer text is not submitted. The extension removes only the wake and one transport blank-line separator, preserves operator draft bytes including invisible marks and leading/trailing newlines, and resends the wake alone through omp's prompt-starting message API. -Recovery is bounded to three resubmission attempts per wake; another `agent_end` is needed to schedule another check. -A wake restored by Alt+Up while idle without `agent_end` is not resubmitted, and rare credential loss during recovery can reject resubmission after the editable copy is removed; the durable queue and shutdown handoff retain the wake, the existing parent stalled-loop alarm reports either stall for endpoint-recorded local secondmates, and consumption-confirmed removal remains follow-up `fm-omp-wake-recovery-rollback`. +Recovery is bounded to three resubmission attempts per tracked wake; recovering an editor segment relies on a later `agent_end` for the next check, while stuck-queue resubmissions schedule their own follow-up checks. +A queue that is still full once omp has stayed idle through that wait is stuck, because omp's follow-up gate refused to start a turn for it. +A wake sitting in the composer is then recovered as above. +If no eligible unchanged tracked wake can be recovered from the composer, the check resubmits the oldest unconsumed wake with attempts remaining through the same prompt-starting API without altering the editor. +That prompt starts a turn and flushes the queue behind it, so the wake can reach main twice and the idempotent drain absorbs the repeat. +Delivery is proven by the accepted user `message_start` carrying the wake; if three resubmissions of every unconsumed wake do not produce it, the extension reports the wait once through omp's own notification and leaves the durable queue and shutdown handoff to the parent's stalled-loop alarm. +Alt+Up does not itself schedule a recovery check; a tracked wake dequeued after the existing check has finished can remain until a later `agent_end`. +Rare credential loss during recovery can reject resubmission after the editable copy is removed; the durable queue and shutdown handoff retain the wake, the existing parent stalled-loop alarm reports either stall for endpoint-recorded local secondmates, and consumption-confirmed removal remains follow-up `fm-omp-wake-recovery-rollback`. [Architecture](architecture.md#event-driven-supervision) owns the parent no-draft boundary, secondmate stalled-queue escalation, and idle-ring eligibility. `tests/fm-omp-harness.test.sh` covers restored-wake matching, editor normalization, draft preservation, bounded recovery, pending-wake retention across cancelled preparation without `agent_end`, later completed draft turns, and replacement handoff until accepted user `message_start`, plus identical wakes across preparation plus accepted-message callbacks, streaming delivery, and session replacement. +It also covers the wait on queued messages: a queue that drains, a restored wake behind a queue that never drains, a wake stuck in the queue itself, and resubmissions that start no turn. +Deterministic lifecycle checks prove that episodes after session replacement, queue drainage observed by recovery checks or polling while exhausted wakes remain tracked, or a running-turn observation each wait all fifteen checks before retrying and warn once after exhausting their attempts. +Separate cases isolate accepted wake messages, ordinary user messages, and turn completion between checks. +Recovery delays and wait limits are fixed in production; the fixture controls timer scheduling in its Node host to accelerate bounded waits and isolate polling scenarios. The opt-in live guard and its evidence limits are recorded in [omp injected text through Herdr](verification/runtime-backends.md#2026-10-06-omp-injected-text-through-herdr). +### omp stranded wake text + +An idle omp raises no event when text lands in its composer, so wake text that older wiring typed there and never submitted stayed pending until someone pressed Enter, and a restart that needs an empty composer was refused. +Once the extension owns the watch for the lock-holding session, at session start or through `fm_watch_arm_omp`, `.omp/extensions/fm-primary-omp-watch.ts` polls the editor every three seconds. +A poll acts only when omp is idle and the editor holds a complete Firstmate watcher wake: the watcher envelope header from `bin/fm-operational-input.sh`, the wake body, and its fixed closing sentence, bounded by editor edges or omp's blank-line joins, with the leading transport mark present or absent. +The full editor text, including any draft, must match on two consecutive eligible idle polls before a wake is sent. +The extension sends the wake alone through omp's prompt-starting message API, restores the transport mark, and then removes only that wake and one blank-line separator from the editor. +Any other text is left exactly as found: operator drafts, a structurally edited or truncated wake, and a wake this extension still tracks, which the bounded restored-wake recovery above owns. +Before template-based recovery, every tracked wake must be accounted for by its own exact composer segment, allowing only omission of the leading transport mark. If any tracked copy is missing or edited, polling waits for exact restored recovery or accepted-message consumption to resolve ownership rather than treating an exact-match refusal as permission to submit a template match. +Accepted residual: with older-wiring wakes A and B in the composer, the poll sends A and removes its segment; if Escape cancels A's preparation before the user `message_start`, A stays tracked but absent, so the poll does not send B until an owner round removes the stall. Edited-copy and duplicate protections remain in force. The cure is assigned to the wake-replay option D round. +Once the tracked-copy ownership gate is satisfied, the scan skips unchanged tracked wakes and stranded text that has exhausted its attempts, so neither can block a later eligible wake; recovering that later wake preserves the skipped text and operator drafts. +Delivery is bounded to three attempts per distinct wake text, and a wake sent this way is tracked until an accepted user `message_start` consumes it. +`tests/fm-omp-harness.test.sh` covers delivery with and without the transport mark, operator drafts on both sides, several wakes, text that is not a whole wake, a running turn, text still changing, the attempt bound, edited tracked copies with polling active, duplicate ownership, and later eligible wakes behind exhausted tracked or stranded text. +The live guard's stranded-wake step and its evidence are recorded in [omp stranded wake text and the queue panel](verification/runtime-backends.md#2026-10-08-omp-stranded-wake-text-and-the-queue-panel). + ### Cursor stop hook Cursor's `.cursor/hooks.json` `stop` hook (`bin/fm-turnend-guard-cursor.sh`) owns routine tokenless re-arm for a Cursor primary. diff --git a/tests/fm-omp-harness.test.sh b/tests/fm-omp-harness.test.sh index f4106c35124..535d5552cc7 100755 --- a/tests/fm-omp-harness.test.sh +++ b/tests/fm-omp-harness.test.sh @@ -1431,7 +1431,30 @@ SH FM_WATCH_REARM_RETRY_LIMIT=1 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 \ SCENARIO="$scenario" EXT="$repo/.omp/extensions/fm-primary-omp-watch.ts" node --input-type=module >"$home/scenario.out" 2>&1 <<'EOF' import { pathToFileURL } from "node:url"; -import { writeFileSync, mkdirSync, readFileSync, existsSync } from "node:fs"; +import { writeFileSync, mkdirSync, readFileSync, existsSync, unlinkSync } from "node:fs"; +const realSetTimeout = globalThis.setTimeout; +const realSetInterval = globalThis.setInterval; +const realClearTimeout = globalThis.clearTimeout; +const manualQueue = process.env.SCENARIO === "queue-episodes"; +const restoreChecks = new Set(); +let pollCheck = null; +const fastRestore = process.env.SCENARIO.startsWith("pending-") || process.env.SCENARIO.startsWith("ownership-") || process.env.SCENARIO.startsWith("blocked-") || ["limit-polled", "duplicates-edited-polled"].includes(process.env.SCENARIO); +const pollWake = process.env.SCENARIO.startsWith("stranded-") || process.env.SCENARIO.startsWith("ownership-") || process.env.SCENARIO.startsWith("blocked-") || ["limit-polled", "duplicates-edited-polled", "editor-normalized-edited", "prepended", "appended", "appended-newline", "prepended-mark", "appended-mark", "internal-mark", "edited"].includes(process.env.SCENARIO); +globalThis.setTimeout = (callback, delay, ...args) => { + if (manualQueue && delay === 2000) { + const timer = { callback: () => callback(...args), unref() {} }; + restoreChecks.add(timer); + return timer; + } + return realSetTimeout(callback, delay === 2000 && fastRestore ? 100 : delay, ...args); +}; +globalThis.clearTimeout = (timer) => { + if (!restoreChecks.delete(timer)) realClearTimeout(timer); +}; +globalThis.setInterval = (callback, delay, ...args) => { + if (manualQueue && delay === 3000) pollCheck = () => callback(...args); + return realSetInterval(callback, delay === 3000 ? (pollWake ? 100 : 3600000) : delay, ...args); +}; writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); const handlers = new Map(); const commands = new Map(); let tool = null; const sent = []; const turns = []; let receiveWake; @@ -1450,7 +1473,13 @@ const pi = { } if (process.env.SCENARIO === "failed-send") throw new Error("fixture send rejected"); // omp's idle auto-continue refuses an explicit follow-up behind an advisor tail. - if (process.env.SCENARIO === "custom-tail" || process.env.SCENARIO.startsWith("idle-")) { + if (process.env.SCENARIO.startsWith("pending-") && !o?.deliverAs && process.env.SCENARIO !== "pending-unflushable") { + // omp's prompt flow starts a turn and flushes whatever sat in its queue. + queued = false; + handlers.get("before_agent_start")({ prompt: m }, ctx); + handlers.get("message_start")({ message: { role: "user", content: m } }, ctx); + } + if (process.env.SCENARIO === "custom-tail" || process.env.SCENARIO.startsWith("idle-") || process.env.SCENARIO.startsWith("stranded-")) { if (o?.deliverAs) { queued = true; return undefined; } if (idle) { turns.push({ prompt: m, tail: transcript.at(-1) }); @@ -1474,12 +1503,17 @@ const composer = { }; // Restored-wake scenarios deliver behind a running turn; idle-* scenarios // deliver to a lane that already went idle. -let idle = process.env.SCENARIO.startsWith("idle-"); let queued = false; +let idle = process.env.SCENARIO.startsWith("idle-") || process.env.SCENARIO.startsWith("stranded-"); let queued = false; +const notices = []; const ctx = { hasUI: true, isIdle: () => { if (process.env.SCENARIO === "idle-stale-context" || process.env.SCENARIO === "idle-recovery-stale") throw new Error("stale context"); return idle; }, hasPendingMessages: () => queued, - ui: { getEditorText: () => composer.text, setEditorText: (t) => { composer.sets.push(t); composer.text = t; } }, + ui: { + getEditorText: () => composer.text, + setEditorText: (t) => { composer.sets.push(t); composer.text = t; }, + notify: (m, level) => { notices.push({ m, level }); }, + }, }; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const mod = await import(pathToFileURL(process.env.EXT).href); @@ -1538,6 +1572,115 @@ else for (let i = 0; i < 600 && sent.length < expectedWakes; i += 1) await sleep if (sent.length !== expectedWakes) throw new Error(`expected ${expectedWakes} wakes, saw ${sent.length}`); const wake = sent[0].m; if (recovery && !wake.includes("signal: omp-restore done")) throw new Error(`recovery did not deliver its actionable close: ${wake}`); +if (process.env.SCENARIO.startsWith("stranded-")) { + // Wake text left unsubmitted in an idle composer by older wiring. The arm + // fixture's own wake (delivered and consumed as an idle turn) supplies the + // wire shape; variants carry other messages. + const wakeWith = (message) => wake.replace("signal: omp-restore done", message); + const waitFor = async (check, ms = 4000) => { for (let i = 0; i < ms / 50 && !check(); i += 1) await sleep(50); return check(); }; + const turnEnds = () => { idle = true; }; + turnEnds(); + const delivered = (n, text) => sent.length === n && sent[n - 1].m === text && sent[n - 1].o?.deliverAs === undefined; + switch (process.env.SCENARIO) { + case "stranded-alone": + case "stranded-unmarked": { + const typed = wakeWith("signal: stranded-one"); + composer.text = process.env.SCENARIO === "stranded-alone" ? typed : typed.slice(1); + if (!await waitFor(() => sent.length === 2)) throw new Error("wake text left in an idle composer was never delivered"); + if (!delivered(2, typed)) throw new Error(`the stranded wake was not sent whole through the prompt flow: ${JSON.stringify(sent[1])}`); + if (composer.text !== "") throw new Error(`the composer still holds text after delivery: ${JSON.stringify(composer.text)}`); + if (turns.length !== 2) throw new Error("delivery did not start the wake's own turn"); + turnEnds(); + await sleep(700); + if (sent.length !== 2) throw new Error("a consumed stranded wake was sent again"); + break; + } + case "stranded-draft-after": + case "stranded-draft-before": { + const typed = wakeWith("signal: stranded-draft"); + const after = "\noperator\u2063 draft\n\n"; + const before = "\n\nbefore\u2063 draft\n\n"; + const after_case = process.env.SCENARIO === "stranded-draft-after"; + composer.text = after_case ? `${typed}\n\n${after}` : `${before}\n\n${typed}`; + if (!await waitFor(() => sent.length === 2)) throw new Error("a stranded wake beside an operator draft was never delivered"); + if (!delivered(2, typed)) throw new Error("the stranded wake was not sent alone"); + if (composer.text !== (after_case ? after : before)) throw new Error(`operator draft bytes changed: ${JSON.stringify(composer.text)}`); + break; + } + case "stranded-two": { + const first = wakeWith("signal: stranded-first"); + const second = wakeWith("signal: stranded-second"); + composer.text = `${first}\n\n${second}`; + if (!await waitFor(() => sent.length === 2)) throw new Error("the first stranded wake was never delivered"); + if (!delivered(2, first) || composer.text !== second) throw new Error(`the first wake was not sent alone: ${JSON.stringify(composer.text)}`); + turnEnds(); + if (!await waitFor(() => sent.length === 3)) throw new Error("the second stranded wake was never delivered"); + if (!delivered(3, second) || composer.text !== "") throw new Error("the second wake was not sent alone"); + break; + } + case "stranded-human": { + const refusals = [ + "FIRSTMATE_OP: v1 watcher: typed by a person", + `Please explain this: ${wake}`, + `${wake} and then do this`, + `${wake}\n`, + wake.replace("Watcher continuity", "Watcher edited"), + wake.slice(0, -12), + "bin/fm-wake-drain.sh first and handle the queued wake. Watcher continuity is extension-owned.", + "an unrelated operator draft", + ]; + for (const text of refusals) { + composer.text = text; + const expected = composer.text; + const sets = composer.sets.length; + await sleep(600); + if (sent.length !== 1 || composer.sets.length !== sets || composer.text !== expected) throw new Error(`text that is not a whole wake was submitted or changed: ${JSON.stringify(text)}`); + } + break; + } + case "stranded-busy": { + const typed = wakeWith("signal: stranded-busy"); + idle = false; + composer.text = typed; + await sleep(600); + if (sent.length !== 1 || composer.text !== typed) throw new Error("a running turn was disturbed"); + turnEnds(); + if (!await waitFor(() => sent.length === 2)) throw new Error("the wake was not delivered once the turn ended"); + if (!delivered(2, typed) || composer.text !== "") throw new Error("the wake was not sent alone after the turn ended"); + break; + } + case "stranded-settling": { + for (let i = 0; i < 8; i += 1) { + composer.text = wakeWith(`signal: stranded-moving-${i}`); + await sleep(40); + } + if (sent.length !== 1) throw new Error("wake text still changing was submitted"); + if (!await waitFor(() => sent.length === 2)) throw new Error("settled wake text was never delivered"); + break; + } + case "stranded-bounded": + case "stranded-bounded-unmarked": { + // Text that keeps coming back must not turn delivery into a loop. + const typed = wakeWith("signal: stranded-again"); + for (let i = 0; i < 6; i += 1) { + composer.text = typed; + await sleep(500); + turnEnds(); + } + if (sent.length !== 4) throw new Error(`delivery was not bounded to three attempts: ${sent.length}`); + const later = wakeWith("signal: stranded-later"); + const exhausted = process.env.SCENARIO === "stranded-bounded" ? typed : typed.slice(1); + composer.text = `${exhausted}\n\n${later}`; + if (!await waitFor(() => sent.length === 5)) throw new Error("an exhausted stranded wake blocked a later eligible wake"); + if (!delivered(5, later) || composer.text !== exhausted) throw new Error("recovering a later wake changed the exhausted wake"); + break; + } + default: + throw new Error(`unknown stranded scenario ${process.env.SCENARIO}`); + } + await handlers.get("session_shutdown")({}, ctx); + process.exit(0); +} if (["idle-stale-context", "idle-recovery-stale", "idle-recovery-busy", "idle-recovery-missing"].includes(scenario)) { if (sent[0].o?.deliverAs !== "followUp" || !queued || turns.length !== 0) throw new Error("delivery without positive idle evidence must remain a follow-up"); await handlers.get("session_shutdown")({}, ctx); @@ -1572,6 +1715,135 @@ const settle = async () => { await handlers.get("agent_end")({ type: "agent_end" const same = (item) => item.m === wake && item.o?.deliverAs === undefined; switch (process.env.SCENARIO) { + case "queue-episodes": { + const check = () => { + if (restoreChecks.size !== 1) throw new Error(`expected one recovery check, saw ${restoreChecks.size}`); + const timer = restoreChecks.values().next().value; + restoreChecks.delete(timer); + timer.callback(); + return timer.callback; + }; + const episode = (label, attempts = 3) => { + const count = sent.length; + const warnings = notices.length; + const content = sent.at(-1).m; + queued = true; + idle = true; + for (let i = 0; i < 15; i += 1) { + check(); + if (sent.length !== count || notices.length !== warnings) throw new Error(`${label}: recovery acted before fifteen checks`); + } + for (let i = 1; i <= attempts; i += 1) { + check(); + if (sent.length !== count + i || sent.at(-1).m !== content || sent.at(-1).o?.deliverAs !== undefined || notices.length !== warnings) throw new Error(`${label}: recovery did not submit exactly ${attempts} prompt attempts`); + } + const recheck = check(); + const notice = notices.at(-1); + if (notices.length !== warnings + 1 || notice.level !== "warning" || !notice.m.includes("wake not delivered")) throw new Error(`${label}: exhausted episode did not warn`); + recheck(); + if (sent.length !== count + attempts || notices.length !== warnings + 1) throw new Error(`${label}: recovery or warning repeated`); + if (composer.text !== "operator draft" || composer.sets.length !== 0) throw new Error(`${label}: recovery disturbed the composer`); + }; + const nextWake = async () => { + const dir = `${process.env.FM_HOME}/state/extensions/omp-primary-watch`; + mkdirSync(dir, { recursive: true }); + writeFileSync(`${dir}/session-replacement-actionable.json`, "invalid"); + idle = false; + const count = sent.length; + await tool.execute(); + unlinkSync(`${dir}/session-replacement-actionable.json`); + if (sent.length !== count + 1 || sent.at(-1).o?.deliverAs !== "followUp") throw new Error("expected a fresh queued failure wake"); + }; + composer.text = "operator draft"; + episode("first session"); + await handlers.get("session_shutdown")({}, ctx); + idle = false; + const beforeReplacement = sent.length; + await handlers.get("session_start")({}, ctx); + for (let i = 0; i < 60 && sent.length === beforeReplacement; i += 1) await sleep(100); + if (sent.length !== beforeReplacement + 1 || sent.at(-1).m !== wake || sent.at(-1).o?.deliverAs !== "followUp") throw new Error("replacement did not replay its unconsumed wake"); + await nextWake(); + episode("replacement session"); + + await nextWake(); + idle = true; + queued = false; + check(); + await nextWake(); + episode("queue drained observed by recovery", 6); + + queued = false; + pollCheck(); + await nextWake(); + episode("queue drained observed by polling"); + + await nextWake(); + check(); + await nextWake(); + episode("running turn observed by recovery", 6); + + idle = false; + pollCheck(); + await nextWake(); + episode("running turn observed by polling"); + + await nextWake(); + await handlers.get("before_agent_start")({ prompt: wake }, ctx); + await handlers.get("message_start")({ message: { role: "user", content: [{ type: "text", text: wake }] } }, ctx); + episode("short accepted wake turn before completion"); + + await nextWake(); + await handlers.get("before_agent_start")({ prompt: "continue work" }, ctx); + await handlers.get("message_start")({ message: { role: "user", content: "continue work" } }, ctx); + episode("short accepted ordinary turn before completion"); + + idle = false; + await handlers.get("before_agent_start")({ prompt: "continue work" }, ctx); + await handlers.get("message_start")({ message: { role: "assistant", content: "done" } }, ctx); + idle = true; + await handlers.get("agent_end")({}, ctx); + await nextWake(); + episode("short turn completion between checks"); + break; + } + case "ownership-marked": + case "ownership-unmarked": { + const before = "\n\nbefore\u2063 draft\n\n"; + const after = "\noperator\u2063 draft\n\n"; + const edited = (process.env.SCENARIO === "ownership-marked" ? wake : bare).replace("signal:", "edited:"); + composer.text = `${before}\n\n${edited}\n\n${after}`; + const original = composer.text; + await settle(); + if (sent.length !== 1 || composer.sets.length !== 0 || composer.text !== original) throw new Error("template matching took ownership of an edited tracked wake"); + composer.text = `${before}\n\n${process.env.SCENARIO === "ownership-marked" ? wake : bare}\n\n${after}`; + await settle(); + if (sent.length !== 2 || !same(sent[1]) || composer.text !== `${before}\n\n${after}`) throw new Error("exact restored recovery no longer preserved both drafts"); + break; + } + case "ownership-missing": { + const older = wake.replace("signal: omp-restore done", "signal: older-wiring"); + composer.text = older; + await settle(); + if (sent.length !== 1 || composer.sets.length !== 0 || composer.text !== older) throw new Error("an unaccounted tracked wake allowed template submission"); + await handlers.get("message_start")({ message: { role: "user", content: wake } }, ctx); + for (let i = 0; i < 40 && sent.length < 2; i += 1) await sleep(50); + if (sent.length !== 2 || sent[1].m !== older || sent[1].o?.deliverAs !== undefined || composer.text !== "") throw new Error("resolved ownership did not release the stranded wake"); + break; + } + case "duplicates-edited-polled": { + if (sent[1].m !== wake) throw new Error("expected two identical tracked wakes"); + const edited = bare.replace("signal:", "edited:"); + composer.text = `${wake}\n\n${edited}`; + await settle(); + if (sent.length !== 3 || !same(sent[2]) || composer.text !== edited) throw new Error("an edited duplicate bypassed restored ownership"); + await handlers.get("message_start")({ message: { role: "user", content: wake } }, ctx); + await sleep(600); + if (sent.length !== 3 || composer.text !== edited) throw new Error("consuming one duplicate authorized the edited duplicate"); + composer.text = bare; + await settle(); + if (sent.length !== 4 || !same(sent[3]) || composer.text !== "") throw new Error("the unchanged duplicate was no longer recoverable"); + break; + } case "duplicates": case "duplicates-handoff": case "duplicates-streaming": { @@ -1786,6 +2058,91 @@ switch (process.env.SCENARIO) { if (sent.length !== 1 || composer.sets.length !== 0) throw new Error("a wake with messages still queued was submitted again"); break; } + case "pending-persistent": { + // Messages queued behind an idle session never drain; the restored wake in + // the composer still has to reach main once the bounded wait is over. + composer.text = wake; queued = true; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + for (let i = 0; i < 60 && sent.length < 2; i += 1) await sleep(50); + if (sent.length !== 2 || !same(sent[1]) || composer.text !== "") throw new Error(`a restored wake behind a stuck queue was left in the composer: ${sent.length} sent, composer ${JSON.stringify(composer.text)}`); + if (queued || notices.length !== 0) throw new Error("the stuck queue was not flushed by the resubmission, or a delivered wake was reported"); + break; + } + case "pending-drains": { + // A queue that drains inside the wait is left to omp: the wake is recovered + // from the composer only after it is gone, and nothing is reported. + composer.text = wake; queued = true; + setTimeout(() => { queued = false; }, 150); + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + for (let i = 0; i < 60 && sent.length < 2; i += 1) await sleep(50); + if (sent.length !== 2 || !same(sent[1]) || composer.text !== "" || notices.length !== 0) throw new Error("a restored wake was not recovered once the queue drained"); + break; + } + case "pending-stuck": { + // The wake itself sits in omp's queue (omp refused to start a turn for it) + // and nothing is restored to the composer. + queued = true; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + for (let i = 0; i < 60 && sent.length < 2; i += 1) await sleep(50); + if (sent.length !== 2 || !same(sent[1])) throw new Error(`a wake stuck in omp's idle queue was never sent through the prompt flow: ${sent.length} sent`); + if (queued || composer.sets.length !== 0 || notices.length !== 0) throw new Error("the stuck queue was not flushed, or the composer was disturbed"); + await sleep(600); + if (sent.length !== 2) throw new Error("a delivered wake was sent again"); + break; + } + case "pending-unflushable": { + // The resubmissions do not start a turn: delivery is not proven, so the + // wait is reported once and the bound holds. + queued = true; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + await sleep(2800); + if (sent.length !== 4 || !sent.slice(1).every(same)) throw new Error(`resubmission was not bounded to three attempts: ${sent.length}`); + if (notices.length !== 1 || notices[0].level !== "warning" || !notices[0].m.includes("wake not delivered")) throw new Error(`the undelivered wake was not reported once: ${JSON.stringify(notices)}`); + await sleep(600); + if (sent.length !== 4 || notices.length !== 1) throw new Error("the report or the resubmission repeated"); + break; + } + case "limit-polled": { + // The stranded-wake poll must not hand a wake the restored-wake recovery has + // given up on another set of attempts. + for (let i = 0; i < 6; i += 1) { + composer.text = wake; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + await sleep(500); + } + if (sent.length !== 4) throw new Error(`recovery was not bounded to three resubmissions with the poll running: ${sent.length}`); + break; + } + case "blocked-tracked-marked": + case "blocked-tracked-unmarked": { + for (let i = 0; i < 3; i += 1) { + composer.text = wake; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + await sleep(500); + } + if (sent.length !== 4) throw new Error("tracked recovery did not reach its attempt limit"); + const exhausted = process.env.SCENARIO === "blocked-tracked-marked" ? wake : bare; + const older = wake.replace("signal: omp-restore done", "signal: older-wiring"); + const before = "\n\nbefore\u2063 draft\n\n"; + const after = "\noperator\u2063 draft\n\n"; + composer.text = `${before}\n\n${exhausted}\n\n${older}\n\n${after}`; + for (let i = 0; i < 40 && sent.length < 5; i += 1) await sleep(50); + if (sent.length !== 5 || sent[4].m !== older || sent[4].o?.deliverAs !== undefined) throw new Error("an exhausted tracked wake blocked a later stranded wake"); + if (composer.text !== `${before}\n\n${exhausted}\n\n${after}`) throw new Error("recovering a later wake changed tracked wake or draft bytes"); + break; + } + case "restore-window": { + // The run ends well after omp accepted the wake. The editor is read two full + // seconds after the run ended, not two seconds after the wake was accepted. + await sleep(1200); + composer.text = wake; + await handlers.get("agent_end")({ type: "agent_end" }, ctx); + await sleep(1400); + if (sent.length !== 1 || composer.text !== wake) throw new Error("the restored wake was taken before the run-end settle time had passed"); + for (let i = 0; i < 40 && sent.length < 2; i += 1) await sleep(100); + if (sent.length !== 2 || !same(sent[1]) || composer.text !== "") throw new Error("the restored wake was not submitted again after the settle time"); + break; + } case "elsewhere": { composer.text = "an unrelated operator draft"; await settle(); @@ -1809,9 +2166,18 @@ EOF return "$status" } +test_watch_queue_episodes_reset_across_sessions_and_completion() { + local out status + out=$(run_watch_restore_scenario queue-episodes) + status=$? + expect_code 0 "$status" "omp stuck-queue episode lifecycle: $out" + [ -z "$out" ] || fail "omp stuck-queue episode lifecycle printed output: $out" + pass ".omp watch extension: isolated replacement, queue drain, running observations, accepted messages, and turn completion each restart the fifteen-check wait and once-only warning" +} + test_watch_extension_resubmits_a_wake_omp_restored_to_the_composer() { local scenario out status - for scenario in duplicates duplicates-handoff duplicates-streaming preparation-cancelled preparation-handoff editor-normalized editor-normalized-message editor-normalized-edited nonpending failed-send sync-consumed consumed normalized-consumed draft custom-tail idle-empty idle-draft idle-stale-context idle-recovery-timer idle-recovery-factory idle-recovery-live-factory idle-recovery-tool idle-recovery-command idle-recovery-forwarded idle-recovery-tool-context idle-recovery-command-context idle-recovery-busy idle-recovery-stale idle-recovery-missing draft-before draft-after-bytes draft-before-bytes draft-both prepended appended appended-newline prepended-mark appended-mark internal-mark edited alone alone-marked busy queued elsewhere limit; do + for scenario in duplicates duplicates-handoff duplicates-streaming duplicates-edited-polled preparation-cancelled preparation-handoff editor-normalized editor-normalized-message editor-normalized-edited nonpending failed-send sync-consumed consumed normalized-consumed draft custom-tail idle-empty idle-draft idle-stale-context idle-recovery-timer idle-recovery-factory idle-recovery-live-factory idle-recovery-tool idle-recovery-command idle-recovery-forwarded idle-recovery-tool-context idle-recovery-command-context idle-recovery-busy idle-recovery-stale idle-recovery-missing stranded-alone stranded-unmarked stranded-draft-after stranded-draft-before stranded-two stranded-human stranded-busy stranded-settling stranded-bounded stranded-bounded-unmarked ownership-marked ownership-unmarked ownership-missing blocked-tracked-marked blocked-tracked-unmarked pending-persistent pending-drains pending-stuck pending-unflushable limit-polled restore-window draft-before draft-after-bytes draft-before-bytes draft-both prepended appended appended-newline prepended-mark appended-mark internal-mark edited alone alone-marked busy queued elsewhere limit; do out=$(run_watch_restore_scenario "$scenario") status=$? expect_code 0 "$status" "omp watch restore scenario $scenario: $out" @@ -2285,6 +2651,7 @@ test_watch_extension_keeps_the_arm_without_the_file_or_with_off test_watch_extension_replays_a_host_only_boundary_across_replacement test_watch_extension_delivers_a_split_host_close_whole test_watch_extension_resubmits_a_wake_omp_restored_to_the_composer +test_watch_queue_episodes_reset_across_sessions_and_completion test_primary_extensions_ignore_a_descendant_session test_turnend_marker_follows_the_lock_owner_at_turn_boundaries test_watch_extension_heals_a_generation_stopped_without_a_successor diff --git a/tests/fm-omp-wake-restore-live-e2e.test.sh b/tests/fm-omp-wake-restore-live-e2e.test.sh index 2b964e5d567..bbcb22827c3 100755 --- a/tests/fm-omp-wake-restore-live-e2e.test.sh +++ b/tests/fm-omp-wake-restore-live-e2e.test.sh @@ -25,9 +25,18 @@ # idle lane sat unread until someone pressed Enter. The watch extension must # start that wake's own turn and leave an operator draft unsent. This step # drives a scripted local model and spends no tokens. +# 5. Wake text an older wiring typed into an idle composer and never submitted +# sat there until someone pressed Enter, and a lane restart that needs an +# empty composer was refused. The watch extension must deliver that wake as +# its own turn, clear only that wake, and leave an operator draft beside it +# exactly as typed. Same scripted model, no tokens. +# The lab-only probe records every change of the editor text: a restored wake sits in +# the composer for only a moment, and one rendered read takes seconds on a busy +# host, so the pane alone cannot be relied on to catch it. # Steps 1-3 submit model prompts, so the guard is opt-in; it fails naming omp # and `omp --version`. Refresh docs/verification/runtime-backends.md -# ("omp injected text" and "omp idle wake") from its output after any omp upgrade. +# ("omp injected text", "omp idle wake", and "omp stranded wake text") from its +# output after any omp upgrade. # Every Herdr call, including adapter calls, is routed through bin/fm-herdr-lab.sh. set -u unset FM_HOME FM_ROOT_OVERRIDE FM_STATE_OVERRIDE FM_CONFIG_OVERRIDE FM_DATA_OVERRIDE @@ -130,6 +139,41 @@ while IFS= read -r path; do done < "$PROJECT/.omp/extensions/fm-lab-queue-probe.ts" <<'TS' +import { appendFileSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +const out = resolve(dirname(fileURLToPath(import.meta.url)), "../../state/.lab-queue-probe"); +const editorLog = resolve(dirname(fileURLToPath(import.meta.url)), "../../state/.lab-editor-log"); +export default function (pi: { on?: (event: string, handler: (event: any, ctx: any) => unknown) => void }) { + let ctx: any = null; + const remember = (_event: unknown, context: unknown) => { + if (typeof context === "object" && context !== null) ctx = context; + }; + for (const event of ["session_start", "before_agent_start", "message_start", "agent_end"]) pi.on?.(event, remember); + let lastEditor = ""; + const timer = setInterval(() => { + try { + const editor = String(ctx?.ui?.getEditorText?.() ?? ""); + if (editor !== lastEditor) { + lastEditor = editor; + appendFileSync(editorLog, `${Math.floor(Date.now() / 1000)} ${JSON.stringify(editor)}\n`); + } + } catch {} + try { + writeFileSync(out, `${Math.floor(Date.now() / 1000)} queued=${ctx?.hasPendingMessages?.() === true ? 1 : 0} idle=${ctx?.isIdle?.() === true ? 1 : 0}\n`); + } catch {} + }, 100); + timer.unref(); +} +TS # The launching code root must be distinct from the secondmate home; the # ordinary spawn interface deliberately refuses its own repository as a home. cp -R "$PROJECT" "$CODE_ROOT" || fail "could not prepare the isolated launching code root" @@ -205,6 +249,10 @@ start_omp() { TERMINAL_CONTROL_PID=$! wait_for 120 is_idle || { screen >&2; fail "$SUBJECT never published settled task evidence for $label"; } sleep 2 + # Native idle can precede the initial launch-brief turn. Do not type the arm + # request until omp's own context confirms that turn has settled. + wait_for 120 probe_says_state "$PROJECT/state/.lab-queue-probe" ' idle=1' \ + || { screen >&2; fail "$SUBJECT: the launch-brief turn did not settle for $label"; } send_text 'Call the fm_watch_arm_omp tool exactly once now, then reply with only the word ARMED.' sleep 1 send_key Enter @@ -233,38 +281,64 @@ queue_wake() { : > "$PROJECT/state/$WAKE_TASK.meta" printf 'done: wake lab signal %s\n' "$WAKE_TASK" > "$PROJECT/state/$WAKE_TASK.status" wait_for 60 wake_is_queued \ - || { screen >&2; fail "$SUBJECT: $WAKE_TASK was not submitted into the running turn's follow-up queue (queue rows: $(queue_rows))"; } + || { screen >&2; fail "$SUBJECT: $WAKE_TASK was not submitted into the running turn's follow-up queue (queue rows: $(queue_rows), probe: $(cat "$PROJECT/state/.lab-queue-probe" 2>/dev/null || echo none))"; } } -wake_is_queued() { - local capture queued - is_busy || return 1 +wake_row_queued() { awk -F '\t' -v key="$WAKE_TASK.status" \ '$3 == "signal" && $4 == key { found = 1 } END { exit !found }' \ - "$PROJECT/state/.wake-queue" 2>/dev/null || return 1 - capture=$(screen) - queued=$(printf '%s\n' "$capture" | awk ' - /After yield.*[1-9][0-9]*/ { in_queue = 1; next } - in_queue && /to edit/ { printf "%s", rows; exit } - in_queue { rows = rows $0 } - ' | tr -d '[:space:]') - case "$queued" in - *"FIRSTMATEWATCHERWAKE:"*"$WAKE_TASK.status"*) ;; - *) return 1 ;; - esac - is_busy && \ - awk -F '\t' -v key="$WAKE_TASK.status" \ - '$3 == "signal" && $4 == key { found = 1 } END { exit !found }' \ - "$PROJECT/state/.wake-queue" 2>/dev/null + "$PROJECT/state/.wake-queue" 2>/dev/null +} + +# omp's own answer through the lab probe, no more than three seconds old. +probe_says_state() { # + local line stamp now + line=$(cat "$1" 2>/dev/null) || return 1 + stamp=${line%% *} + now=$(date +%s) + case "$stamp" in ''|*[!0-9]*) return 1 ;; esac + [ $((now - stamp)) -le 3 ] || return 1 + case "$line" in *"$2"*) return 0 ;; esac + return 1 +} +probe_says_queued() { probe_says_state "$PROJECT/state/.lab-queue-probe" ' queued=1 '; } + +wake_is_queued() { + is_busy || return 1 + wake_row_queued || return 1 + probe_says_queued || return 1 + is_busy && wake_row_queued +} + +# omp's own record that Escape restored a watcher wake into its editor: the probe +# logs every change of the editor text, so a restoration that lasts a moment cannot +# fall between the pane reads below. Any watcher wake counts, not only this task's +# signal: the lane also receives the secondmate's own outcome checks, one of which +# can already be queued when the signal arrives. Every wake is tracked and +# restored alike. RESTORE_BASE marks the log before Escape, and RESTORE_DRAFT +# (when set) must sit in the same editor text as the wake. +RESTORE_BASE=0 +RESTORE_DRAFT= +editor_log_lines() { echo $(( $(wc -l < "$PROJECT/state/.lab-editor-log" 2>/dev/null || echo 0) )); } +editor_log_restored_wake() { + local entries + entries=$(tail -n +"$((RESTORE_BASE + 1))" "$PROJECT/state/.lab-editor-log" 2>/dev/null | grep -F 'FIRSTMATE WATCHER WAKE') || return 1 + [ -z "$RESTORE_DRAFT" ] || printf '%s\n' "$entries" | grep -F -- "$RESTORE_DRAFT" >/dev/null } wake_is_restored() { - local content + local content draft + editor_log_restored_wake && return 0 composer_is pending || return 1 content=$(fm_backend_herdr_composer_content "$TARGET" '') || return 1 content=$(printf '%s' "$content" | tr -d '[:space:]') case "$content" in - *"FIRSTMATEWATCHERWAKE:"*"$WAKE_TASK.status"*) return 0 ;; + *"FIRSTMATEWATCHERWAKE:"*) ;; + *) return 1 ;; + esac + draft=$(printf '%s' "$RESTORE_DRAFT" | tr -d '[:space:]') + case "$content" in + *"$draft"*) return 0 ;; *) return 1 ;; esac } @@ -278,6 +352,8 @@ interrupt_queued_wake() { sleep 1 fi queue_wake + RESTORE_BASE=$(editor_log_lines) + RESTORE_DRAFT=$draft send_key Escape for ((poll = 0; poll < 50; poll++)); do wake_is_restored && return 0 @@ -310,6 +386,7 @@ interrupt_queued_wake() { IDLE="$LAB/idle" mkdir -p "$IDLE/agent" "$IDLE/home/.omp/extensions" "$IDLE/home/.pi/extensions/lib" "$IDLE/home/bin" "$IDLE/home/state" cp "$PROJECT/.omp/extensions/fm-primary-omp-watch.ts" "$IDLE/home/.omp/extensions/" +cp "$PROJECT/.omp/extensions/fm-lab-queue-probe.ts" "$IDLE/home/.omp/extensions/" cp "$PROJECT/.pi/extensions/lib/fm-operational-input.ts" "$IDLE/home/.pi/extensions/lib/" cp "$PROJECT/.pi/extensions/lib/fm-watch-lifecycle.ts" "$IDLE/home/.pi/extensions/lib/" cp "$PROJECT/bin/fm-operational-input.sh" "$IDLE/home/bin/" @@ -396,14 +473,19 @@ IDLE_PANE=$(lab workspace create --cwd "$IDLE/home" --label idle-wake --no-focus [ -n "$IDLE_PANE" ] || fail "could not create the idle-wake lab pane" # The pid written to state/.lock is the omp the shell execs, so the extension # owns the lock and arms at session_start without a model turn. -lab pane run "$IDLE_PANE" "bash -c 'printf \"%s\\n\" \$\$ > $IDLE/home/state/.lock; exec env FM_HOME=$IDLE/home PI_CODING_AGENT_DIR=$IDLE/agent $REAL_OMP --model lab/m1'" >/dev/null \ +# Use the same box overlay as the spawned lane: the full-editor reader requires +# its status-bearing top and folded bottom borders, not omp's default shape. +lab pane run "$IDLE_PANE" "bash -c 'printf \"%s\\n\" \$\$ > $IDLE/home/state/.lock; exec env FM_HOME=$IDLE/home PI_CODING_AGENT_DIR=$IDLE/agent $REAL_OMP --config $BOX_OVERLAY --model lab/m1'" >/dev/null \ || fail "could not start omp in the idle-wake lab pane" idle_screen() { lab pane read "$IDLE_PANE" --source visible 2>/dev/null || true; } model_saw() { grep -F -- "$1" "$IDLE/model.log" >/dev/null 2>&1; } note_posted() { idle_screen | grep -F 'IDLE-LAB-NOTE' >/dev/null; } +idle_ready() { probe_says_state "$IDLE/home/state/.lab-queue-probe" ' idle=1'; } wait_for 60 test -s "$IDLE/home/state/.omp-watch-extension-loaded" \ || { idle_screen >&2; fail "$SUBJECT: the idle-wake session never loaded the watch extension"; } -sleep 3 +# The loaded marker is written before session_start and does not prove that omp +# accepts input yet; wait for the real context's fresh idle answer instead. +wait_for 120 idle_ready || { idle_screen >&2; fail "$SUBJECT: the idle-wake editor never became ready"; } lab pane send-text "$IDLE_PANE" 'IDLE-LAB-ADVISE reply ack' >/dev/null sleep 1 lab pane send-keys "$IDLE_PANE" Enter >/dev/null @@ -419,6 +501,36 @@ model_saw 'operator draft kept' && fail "$SUBJECT: the operator draft was submit idle_screen | grep -F 'operator draft kept' >/dev/null \ || { idle_screen >&2; fail "$SUBJECT: the operator draft did not stay in the composer"; } pass "live omp idle wake: $SUBJECT started its own turn for a wake that reached an idle lane behind an advisor note and left the operator draft unsent" + +# --------------------------------------------------------------------------- +# Session I, second step: wake text an older wiring typed into the idle composer +# and never submitted. It is built from the operational-input owner's legacy wake +# shape and encoder, typed without Enter, beside an operator draft. +# --------------------------------------------------------------------------- +# shellcheck source=bin/fm-operational-input.sh +. "$ROOT/bin/fm-operational-input.sh" +OLD_WAKE_MESSAGE='stale: old-wiring-lane wake left in the composer' +fm_operational_input_encode watcher "${FM_LEGACY_WATCHER_PREFIX}${OLD_WAKE_MESSAGE}${FM_LEGACY_WATCHER_SUFFIX}" OLD_WAKE \ + || fail "could not encode the older-wiring wake" +idle_composer() { fm_backend_herdr_composer_content "$SESSION:$IDLE_PANE" ''; } +draft_cleared() { + local content + content=$(idle_composer) || return 1 + [ -z "$content" ] +} +lab pane send-keys "$IDLE_PANE" ctrl+u >/dev/null +wait_for 10 draft_cleared \ + || { idle_screen >&2; fail "$SUBJECT: could not clear the idle-wake draft before the stranded-wake step"; } +lab pane send-text "$IDLE_PANE" "${OLD_WAKE}"$'\n\n'"operator draft beside the wake" >/dev/null +wait_for 40 model_saw "$OLD_WAKE_MESSAGE" \ + || { idle_screen >&2; fail "$SUBJECT: wake text left unsent in an idle composer was never delivered"; } +sleep 3 +model_saw 'operator draft beside the wake' && fail "$SUBJECT: the operator draft was submitted with the stranded wake" +draft=$(idle_composer) \ + || { idle_screen >&2; fail "$SUBJECT: could not read the full composer after the stranded wake was delivered"; } +[ "$draft" = 'operator draft beside the wake' ] \ + || { idle_screen >&2; fail "$SUBJECT: the stranded wake recovery did not leave exactly the operator draft, composer now holds: '$draft'"; } +pass "live omp stranded wake: $SUBJECT delivered wake text an older wiring left unsent in an idle composer as its own turn, cleared only that wake, and left the operator draft as typed" lab pane close "$IDLE_PANE" >/dev/null 2>&1 || true # --------------------------------------------------------------------------- @@ -436,6 +548,9 @@ pass "live omp wake restore: $SUBJECT re-submitted a wake that Esc restored to t # An operator draft typed while the wake was queued survives the recovery. wait_for 120 is_idle || fail "the lane did not settle after handling the wake" interrupt_queued_wake 'operator draft kept' +draft_alone() { [ "$(fm_backend_herdr_composer_content "$TARGET" '')" = 'operator draft kept' ]; } +wait_for 60 draft_alone \ + || { screen >&2; fail "$SUBJECT: the wake restored next to an operator draft was never taken out of the composer, which holds: '$(fm_backend_herdr_composer_content "$TARGET" '')'"; } wait_for 90 queue_drained \ || { screen >&2; fail "$SUBJECT: the wake restored next to an operator draft was never submitted again"; } draft=$(fm_backend_herdr_composer_content "$TARGET" '')