From 3108205ea427e181ec784f9fb3991a28496c9d3a Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Wed, 26 Aug 2026 19:58:19 -0700 Subject: [PATCH 1/2] fix(pi): type captain supervision outcomes so main relays them A captain-relevant branch outcome reached main as a bare user message with no marker of origin or required action, written in main's own captain-facing voice, landing in a tail that often already held several such notes. Pi keeps only a custom message's content when it builds the provider request, so customType and display never reach the model and content was the only place that identity could live. Main could not tell an incoming outcome from its own earlier answer and sometimes re-emitted that answer instead of relaying the outcome, losing it. Measured against real Pi 0.84.1 on openai-codex/gpt-5.6-sol: 6 failures in 24 turns, rising to 3 in 6 once one stale answer was already in the tail, which is how one captain conversation saw six identical messages in a row. The same scenario with the outcome typed failed 0 times in 14 turns. Wrap only the captain-verdict note in the branch-outcome operational kind owned by bin/fm-operational-input.sh. Delivery is otherwise unchanged: still display: false, still one triggerTurn follow-up, so the turn remains the single captain-visible outcome and no hidden note is ever shown twice. Routine notes stay plain because their renderer reads the glyph off the front of that same string. An outcome that cannot be encoded degrades to the same instruction as plain text rather than being lost, matching this file's stated failure direction. The existing assertions could not catch this: they pin the sendMessage options and never look at what main receives. Add a portable regression that classifies the delivered payload with the real protocol executable, and a live guard that runs the real Pi SDK's own convertToLlm to prove content is the entire model-visible payload. --- .pi/extensions/fm-branch-supervision.ts | 38 +++++++++++- .pi/extensions/lib/fm-operational-input.ts | 1 + bin/fm-operational-input.sh | 3 +- docs/pi-supervision-branch.md | 1 + tests/fm-operational-input.test.sh | 2 +- tests/fm-pi-branch-extension.test.sh | 40 +++++++++++++ tests/fm-pi-branch-live-e2e.test.sh | 68 ++++++++++++++++++++++ 7 files changed, 150 insertions(+), 3 deletions(-) diff --git a/.pi/extensions/fm-branch-supervision.ts b/.pi/extensions/fm-branch-supervision.ts index 5ea4738ebbe..093023fd00b 100644 --- a/.pi/extensions/fm-branch-supervision.ts +++ b/.pi/extensions/fm-branch-supervision.ts @@ -127,6 +127,13 @@ const branchCacheKey = `fm-branch-${createHash("sha256").update(fmHome).digest(" const MIRROR_MESSAGE_CAP = 4000; const MERGE_NOTE_BOAT = "⛵"; +// Carried inside the captain note's own text because that text is the only +// part of a custom message Pi gives the model (see mergeIntoMain). +const CAPTAIN_OUTCOME_INSTRUCTION = + "This is a supervision outcome delivered automatically by the supervision branch. " + + "It was not typed by the captain and it is not your own earlier output. " + + "Relay only this outcome to the captain now, in one short message, in captain outcome language. " + + "Do not restate or repeat any earlier answer."; type MirrorItem = { tag: "captain" | "main"; text: string }; type MirrorCursor = { file: string; index: number }; type Verdict = "routine" | "captain"; @@ -549,6 +556,31 @@ export default function (pi: ExtensionAPI) { // Pi; a crash inside Pi's // own delivery window leaves the outcome durable in the store, where // main's fm_branch_outcomes tool still reads it on demand. + // + // Pi keeps only `content` when it converts a custom message for the model: + // customType, display, and details never reach the provider. A captain note + // therefore has to carry its own identity inside `content`, or main receives + // an unattributed user message written in main's own captain-facing voice + // and cannot tell an incoming outcome from its own earlier answer. When that + // happens main re-emits its previous answer instead of relaying the outcome, + // and the outcome is lost. The typed operational envelope is what makes the + // note self-describing; it stays invisible to the captain because the note + // is never rendered. + // + // Encoding shells out, so it can fail on a broken checkout. This file's + // failure direction applies: an outcome that cannot be typed is still + // delivered, carrying the same instruction as plain text, because an + // untyped outcome main can still read beats an outcome the captain never + // sees. + function captainOutcomeInput(task: string, summary: string): string { + const body = `${CAPTAIN_OUTCOME_INSTRUCTION}\n\n${task}: ${summary}`; + try { + return encodeFirstmateOperationalInput("branch-outcome", body); + } catch { + return body; + } + } + function mergeIntoMain( expectedGeneration: number, seq: string, @@ -559,7 +591,11 @@ export default function (pi: ExtensionAPI) { ): boolean { if (!actingAsOwner(expectedGeneration)) return false; if (verdict === "captain") { - const message = { customType: "fm-branch-merge", content: `${task}: ${summary}`, display: false }; + const message = { + customType: "fm-branch-merge", + content: captainOutcomeInput(task, summary), + display: false, + }; pi.sendMessage(message, { triggerTurn: true, deliverAs: "followUp" }); } else { const message = { customType: "fm-branch-merge", content: `${MERGE_NOTE_BOAT} ${task}: ${summary}`, display: !(task === "fleet" && silent) }; diff --git a/.pi/extensions/lib/fm-operational-input.ts b/.pi/extensions/lib/fm-operational-input.ts index 338312d3f64..ea071ab8720 100644 --- a/.pi/extensions/lib/fm-operational-input.ts +++ b/.pi/extensions/lib/fm-operational-input.ts @@ -13,6 +13,7 @@ export const FIRSTMATE_CURRENT_OPERATIONAL_KINDS = [ "away-supervisor", "from-firstmate", "launch-brief", + "branch-outcome", ] as const; export type FirstmateCurrentOperationalKind = diff --git a/bin/fm-operational-input.sh b/bin/fm-operational-input.sh index 11d6a459d56..d12b406fa73 100755 --- a/bin/fm-operational-input.sh +++ b/bin/fm-operational-input.sh @@ -28,7 +28,7 @@ FM_OPERATIONAL_MARK=$'\xE2\x81\xA3' FM_OPERATIONAL_PREFIX="${FM_OPERATIONAL_MARK}FIRSTMATE_OP: " FM_OPERATIONAL_VERSION=v1 FM_OPERATIONAL_HEADER_PREFIX="${FM_OPERATIONAL_PREFIX}${FM_OPERATIONAL_VERSION} " -FM_OPERATIONAL_KINDS='session-start watcher turn-end-guard away-supervisor launch-brief' +FM_OPERATIONAL_KINDS='session-start watcher turn-end-guard away-supervisor launch-brief branch-outcome' # Compatibility name retained for the away-mode owner and its tests. # shellcheck disable=SC2034 # Public source-library variable used by callers. @@ -204,6 +204,7 @@ Usage: Current construction kinds: session-start watcher turn-end-guard away-supervisor from-firstmate launch-brief + branch-outcome The from-firstmate kind uses its established live-charter-compatible carrier. EOF diff --git a/docs/pi-supervision-branch.md b/docs/pi-supervision-branch.md index 1f22b0a4861..7440206d3fe 100644 --- a/docs/pi-supervision-branch.md +++ b/docs/pi-supervision-branch.md @@ -53,6 +53,7 @@ The branch prompt frames mirrored text as context for judgment, never as instruc Stage one is unchanged: the bash watcher absorbs everything provably fine at zero token cost. Stage two is the branch's verdict on each handled event, reported through its `fm_branch_report` tool: `routine` merges without a follow-up turn, while `captain` merges with exactly one follow-up turn. The follow-up turn a `captain` verdict opens is itself the captain-visible outcome, so its merge note is delivered silently and never printed or rendered in Pi. +Because Pi gives the model only a custom message's `content`, that silent note carries the `branch-outcome` operational kind owned by `bin/fm-operational-input.sh` inside its own text; without it main receives an unattributed message in its own captain-facing voice, cannot tell an incoming outcome from its own earlier answer, and re-emits that answer instead of relaying the outcome. A no-change heartbeat outcome explicitly reported with `task=fleet` and `silent=true` is also delivered silently with no rendered note, while every other `routine` outcome stays rendered with its sailboat prefix. The verdict criteria in the branch prompt mirror the captain-etiquette escalation list; doubt escalates. Main can read the durable outcome store on demand through its `fm_branch_outcomes` tool. diff --git a/tests/fm-operational-input.test.sh b/tests/fm-operational-input.test.sh index 2d1b1c39de3..cdea6d0ed56 100755 --- a/tests/fm-operational-input.test.sh +++ b/tests/fm-operational-input.test.sh @@ -28,7 +28,7 @@ test_current_generic_matrix() { [ "$prefix_hex" = e281a346495253544d4154455f4f503a20 ] \ || fail "current operational prefix lost the landed U+2063 FIRSTMATE_OP bytes: $prefix_hex" - for kind in session-start watcher turn-end-guard away-supervisor launch-brief; do + for kind in session-start watcher turn-end-guard away-supervisor launch-brief branch-outcome; do body="CURRENT_BODY_FOR_${kind}" fm_operational_input_encode "$kind" "$body" encoded \ || fail "could not encode current $kind fixture" diff --git a/tests/fm-pi-branch-extension.test.sh b/tests/fm-pi-branch-extension.test.sh index 1d7741570ac..e4ac23e5eee 100644 --- a/tests/fm-pi-branch-extension.test.sh +++ b/tests/fm-pi-branch-extension.test.sh @@ -645,6 +645,18 @@ if (!sentToMain[2].message.content.includes("task-9: PR https://example.com/pr/9 if (/branch merged|\[routine\]|\[captain\]/.test(sentToMain[2].message.content)) { throw new Error(`captain note still has boilerplate: ${sentToMain[2].message.content}`); } +// What main's model actually receives. Pi keeps only `content` when it turns a +// custom message into a provider message - customType, display, and details are +// all dropped - so `content` IS the delivered payload, and these two files are +// the exact bytes main's model would read. The bash side classifies them with +// the REAL bin/fm-operational-input.sh so the protocol's own executable, not a +// pattern in this test, decides what was delivered. Pi's half of that contract +// is proven separately against the real SDK in fm-pi-branch-live-e2e.test.sh. +writeFileSync(`${home}/state/delivered-captain-note`, sentToMain[2].message.content); +writeFileSync(`${home}/state/delivered-routine-note`, sentToMain[0].message.content); +if (sentToMain.filter((sent) => sent.options.triggerTurn).length !== 1) { + throw new Error("one captain outcome must open exactly one turn on main"); +} // The store (the owned durable contract) holds all three outcomes in order, // and each merged note advanced the read cursor. @@ -747,6 +759,34 @@ EOF *) fail "cache key line missing from driver output: $out" ;; esac pass "branch owns accepted wakes with a stable prefix contract and verdict-driven merge delivery" + + # The delivered captain payload must identify itself to main's model. When it + # did not, main could not tell an incoming outcome from its own earlier answer + # and re-emitted that answer instead of relaying the outcome, silently losing + # it. The real protocol executable is the oracle here: it decides the kind and + # extracts the body, so this asserts delivered behavior rather than a shape + # this test already knows. + local kind body + kind=$(./bin/fm-operational-input.sh kind < "$home/state/delivered-captain-note") \ + || fail "captain outcome reaches main's model as unattributed text the model cannot tell from its own answer" + [ "$kind" = branch-outcome ] \ + || fail "captain outcome delivered as kind '$kind', not branch-outcome" + body=$(./bin/fm-operational-input.sh body < "$home/state/delivered-captain-note") \ + || fail "captain outcome envelope carries no readable body" + case "$body" in + *"task-9: PR https://example.com/pr/9"*) ;; + *) fail "captain outcome body lost the outcome itself: $body" ;; + esac + case "$body" in + *"Relay only this outcome"*"Do not restate or repeat any earlier answer"*) ;; + *) fail "captain outcome body never tells main to relay it instead of repeating: $body" ;; + esac + # The routine note is rendered in the TUI, and its renderer reads the glyph off + # the front of this same string, so it must stay plain text. + if ./bin/fm-operational-input.sh kind < "$home/state/delivered-routine-note" >/dev/null 2>&1; then + fail "routine note must stay plain rendered text, not typed operational input" + fi + pass "a captain outcome reaches main's model as typed, self-describing input while routine notes stay plain" } test_branch_cache_key_is_per_home_stable() { diff --git a/tests/fm-pi-branch-live-e2e.test.sh b/tests/fm-pi-branch-live-e2e.test.sh index 2537285bb4b..098883164ee 100644 --- a/tests/fm-pi-branch-live-e2e.test.sh +++ b/tests/fm-pi-branch-live-e2e.test.sh @@ -457,3 +457,71 @@ if [ "$status" -ne 0 ] || [ "$out" != "EFFORT_OK" ]; then fail "real-SDK effort-pin guard failed against pi-coding-agent $PI_VERSION: $out" fi pass "real Pi SDK $PI_VERSION reports its own supported effort levels and applies an explicit branch effort over a reopened session's recorded level" + +# Fourth probe: the vendor contract the captain-outcome envelope rests on. Pi +# keeps ONLY `content` when it converts a custom message for the provider, so +# `content` is the entire payload main's model receives and is the only place a +# captain outcome can carry its own identity. When it carried none, main could +# not tell an incoming outcome from its own earlier answer and re-emitted that +# answer instead of relaying the outcome. This runs the real SDK's own +# convertToLlm over bytes the REAL protocol encoder produced, then hands the +# model-visible text back to the real parser, proving the delivery path end to +# end instead of assuming it. +captain_payload=$(printf 'relay this\n\ntask-9: PR ready' \ + | "$ROOT/bin/fm-operational-input.sh" encode branch-outcome) \ + || fail "the operational-input owner does not encode the branch-outcome kind" +CAPTAIN_PAYLOAD="$captain_payload" ROUTINE_PAYLOAD="⛵ task-9: worker healthy" \ + DELIVERY_DIR="$TMP_ROOT" PI_PACKAGE_DIR="$PI_PACKAGE_DIR" \ + node --input-type=module > "$TMP_ROOT/delivery-output" 2>&1 <<'EOF' +import { writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +const pkg = resolve(process.env.PI_PACKAGE_DIR); +const { convertToLlm } = await import(pathToFileURL(`${pkg}/dist/index.js`).href); +if (typeof convertToLlm !== "function") { + throw new Error("this Pi no longer exports convertToLlm: the delivery contract is unproven"); +} + +const captainContent = process.env.CAPTAIN_PAYLOAD; +const routineContent = process.env.ROUTINE_PAYLOAD; +const converted = convertToLlm([ + { role: "custom", customType: "fm-branch-merge", content: captainContent, display: false, timestamp: 1 }, + { role: "custom", customType: "fm-branch-merge", content: routineContent, display: true, timestamp: 2 }, +]); +if (converted.length !== 2) { + throw new Error(`Pi no longer delivers one provider message per custom message: ${converted.length}`); +} +for (const message of converted) { + if (message.role !== "user") { + throw new Error(`Pi delivers a custom message as role ${message.role}, not user`); + } + if ("customType" in message || "display" in message) { + throw new Error("Pi now forwards customType or display, so content is no longer the whole payload"); + } +} +const textOf = (message) => + typeof message.content === "string" + ? message.content + : message.content.map((block) => block.text ?? "").join(""); +if (textOf(converted[0]) !== captainContent || textOf(converted[1]) !== routineContent) { + throw new Error("Pi altered custom-message content on the way to the provider"); +} +writeFileSync(`${process.env.DELIVERY_DIR}/live-delivered-captain`, textOf(converted[0])); +writeFileSync(`${process.env.DELIVERY_DIR}/live-delivered-routine`, textOf(converted[1])); +console.log("DELIVERY_OK"); +process.exit(0); +EOF +status=$? +out=$(cat "$TMP_ROOT/delivery-output") +if [ "$status" -ne 0 ] || [ "$out" != "DELIVERY_OK" ]; then + fail "real-SDK custom-message delivery guard failed against pi-coding-agent $PI_VERSION: $out" +fi +delivered_kind=$("$ROOT/bin/fm-operational-input.sh" kind < "$TMP_ROOT/live-delivered-captain") \ + || fail "pi-coding-agent $PI_VERSION delivered the captain outcome as text the protocol cannot type" +[ "$delivered_kind" = branch-outcome ] \ + || fail "pi-coding-agent $PI_VERSION delivered the captain outcome as kind '$delivered_kind'" +if "$ROOT/bin/fm-operational-input.sh" kind < "$TMP_ROOT/live-delivered-routine" >/dev/null 2>&1; then + fail "a routine note survived Pi conversion as typed operational input" +fi +pass "real Pi SDK $PI_VERSION delivers a custom message to the provider as user text carrying only content, so the captain outcome's typed envelope is what reaches the model" From 0d46d987c1600fe9b4f594519bbf1a06ce3b467a Mon Sep 17 00:00:00 2001 From: kunchenguid Date: Wed, 26 Aug 2026 20:07:37 -0700 Subject: [PATCH 2/2] no-mistakes(document): Document typed Pi captain outcomes --- README.md | 2 +- docs/calm.md | 2 +- docs/pi-supervision-branch.md | 8 +++-- docs/verification/runtime-backends.md | 2 ++ tests/fm-pi-branch-extension.test.sh | 50 +++++++++++++++++++++++++++ 5 files changed, 59 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cf5928d79e4..c1f9794195f 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,7 @@ FM_PI_HARNESS=pi-signed pi-signed For Grok, `--trust` is needed once per clone so project hooks and the turn-end guard load; `/hooks-trust` inside Grok works too. For Pi, approve the project trust prompt once per clone on first launch so the tracked `.pi/extensions/*.ts` files auto-load. Pi's `/calm` toggle hides supported transcript chrome, including canonically classified Firstmate operational user rows, and uses a Calm-only animated working boat during active runs while preserving all model context and session data. -The hidden operational inputs remain ordinary user-role messages with unchanged delivery, ordering, authority, persistence, and exports. +Those Calm-hidden operational inputs remain ordinary user-role messages with unchanged delivery, ordering, authority, persistence, and exports. The preference persists for the effective Firstmate home, and toggling it off restores ordinary rendering. [Calm's current behavior and supported limits](docs/calm.md) are separate from its [version-scoped maintainer evidence](docs/calm-mode-feasibility.md). Pi's `/supervision-model` command pins a cheaper model and a shallower reasoning effort for the supervision branch alone, from the eligible models and thinking levels Pi itself reports, and with no pin the branch normally follows your own conversation's model and effort; see the [configuration schema](docs/configuration.md#pi-supervision-branch-model-and-effort-configsupervision-branch-model-configsupervision-branch-effort). diff --git a/docs/calm.md b/docs/calm.md index 360151af399..bac41ae23d9 100644 --- a/docs/calm.md +++ b/docs/calm.md @@ -18,7 +18,7 @@ A mid-turn working note is assistant text in a message the model did not end its Hiding it removes the narration a model emits alongside its tool calls, while the genuine reply that ends a response stays visible. Text that is still streaming is never hidden, because suppressing it would also stop a genuine reply from streaming, so a working note is briefly visible before its row collapses. The narration is hidden only from the live transcript presentation, and remains in the message, model context, session storage, and `/export` artifacts. -The operational inputs remain ordinary user-role messages, while Pi's transcript layout renders their complete rows at zero height. +The operational inputs Calm classifies remain ordinary user-role messages, while Pi's transcript layout renders their complete rows at zero height. The session-start nudge remains on its existing non-displayed custom-message path. Outside Pi's same-name built-in override collision described below, Calm changes presentation only. diff --git a/docs/pi-supervision-branch.md b/docs/pi-supervision-branch.md index 7440206d3fe..f1f04eb2122 100644 --- a/docs/pi-supervision-branch.md +++ b/docs/pi-supervision-branch.md @@ -53,7 +53,9 @@ The branch prompt frames mirrored text as context for judgment, never as instruc Stage one is unchanged: the bash watcher absorbs everything provably fine at zero token cost. Stage two is the branch's verdict on each handled event, reported through its `fm_branch_report` tool: `routine` merges without a follow-up turn, while `captain` merges with exactly one follow-up turn. The follow-up turn a `captain` verdict opens is itself the captain-visible outcome, so its merge note is delivered silently and never printed or rendered in Pi. -Because Pi gives the model only a custom message's `content`, that silent note carries the `branch-outcome` operational kind owned by `bin/fm-operational-input.sh` inside its own text; without it main receives an unattributed message in its own captain-facing voice, cannot tell an incoming outcome from its own earlier answer, and re-emits that answer instead of relaying the outcome. +Because Pi gives the model only a custom message's `content`, that silent note normally carries both a relay instruction and the `branch-outcome` operational kind owned by `bin/fm-operational-input.sh` inside its own text. +This self-description lets main distinguish a new supervision outcome from its own earlier captain-facing answer; without it, main can mistake the outcome for that answer and re-emit the stale answer instead of relaying the outcome. +If envelope encoding fails, the note degrades to the same relay instruction as plain text rather than losing the outcome or opening another turn. A no-change heartbeat outcome explicitly reported with `task=fleet` and `silent=true` is also delivered silently with no rendered note, while every other `routine` outcome stays rendered with its sailboat prefix. The verdict criteria in the branch prompt mirror the captain-etiquette escalation list; doubt escalates. Main can read the durable outcome store on demand through its `fm_branch_outcomes` tool. @@ -86,6 +88,6 @@ What is new is only the attended path: outside away mode, the branch absorbs the ## Verification -Portable regressions: `tests/fm-pi-branch-extension.test.sh` (dispatch, default-on eligibility, main-only classification, eligible-row claim lifecycle, partial pre-drain recheck, fallback, filter, mirror, cache key, persistence, model pin and searchable picker, effort pin), `tests/fm-branch-supervision.test.sh` (prompt stability, store append-only, leases, guards, non-branch-home invariance), the branch-offer, heartbeat-offer, heartbeat-not-ridden-by-a-check, and main-only-check-class tests in `tests/fm-pi-watch-extension.test.sh`, the recovery test in `tests/fm-session-start.test.sh`, and the per-actor consume regression in `tests/fm-wake-queue.test.sh`. -Live guard: `FM_PI_BRANCH_LIVE_E2E=1 tests/fm-pi-branch-live-e2e.test.sh` exercises the real installed Pi SDK with no user credentials and no provider call; run it after every Pi upgrade and record the dated result in [docs/verification/runtime-backends.md](verification/runtime-backends.md). +Portable regressions: `tests/fm-pi-branch-extension.test.sh` (dispatch, default-on eligibility, main-only classification, eligible-row claim lifecycle, partial pre-drain recheck, fallback, filter, mirror, model-visible captain-outcome typing and plain-instruction fallback, cache key, persistence, model pin and searchable picker, effort pin), `tests/fm-branch-supervision.test.sh` (prompt stability, store append-only, leases, guards, non-branch-home invariance), the branch-offer, heartbeat-offer, heartbeat-not-ridden-by-a-check, and main-only-check-class tests in `tests/fm-pi-watch-extension.test.sh`, the recovery test in `tests/fm-session-start.test.sh`, and the per-actor consume regression in `tests/fm-wake-queue.test.sh`. +Live guard: `FM_PI_BRANCH_LIVE_E2E=1 tests/fm-pi-branch-live-e2e.test.sh` exercises the real installed Pi SDK's custom-message conversion and branch-session surfaces with no user credentials and no provider call; run it after every Pi upgrade and record the dated result in [docs/verification/runtime-backends.md](verification/runtime-backends.md). The strict typecheck in `tests/fm-pi-primary-types.test.sh` pins the extension against the installed Pi package. diff --git a/docs/verification/runtime-backends.md b/docs/verification/runtime-backends.md index e5866b088bc..ec9c95911e1 100644 --- a/docs/verification/runtime-backends.md +++ b/docs/verification/runtime-backends.md @@ -963,6 +963,8 @@ Evidence produced 2026-08-25 on macOS 26.5.2 arm64, Node v24.13.1: That case imports the real `SelectList`, `Input`, `fuzzyFilter`, and `DynamicBorder`, renders a 42-row catalog through the real `SelectList` at the visible bound the extension asks for, and fails naming the installed version if Pi stops exporting a primitive or stops bounding what it renders; it skips when no npm package is installed, and the portable stubbed cases in the same file hold the ordering, search, and branch-only-pin behavior everywhere. - Strict typecheck: `tests/fm-pi-primary-types.test.sh` printed `ok - tracked Pi extensions pass strict no-emit typecheck against Pi 0.81.1` with the branch extension and its imported libraries included. This typecheck is also the enforcement for the extension's declared effort vocabulary: its bidirectional assertion against Pi's own `getThinkingLevel` return type fails the moment Pi adds or removes a thinking level, so the runtime list used to reject an unrecognized hand-edited pin cannot drift into a stale Firstmate catalog. +- Custom-message provider conversion: on 2026-08-26, `FM_PI_BRANCH_LIVE_E2E=1 bin/fm-test-run.sh tests/fm-pi-branch-live-e2e.test.sh` against installed `@earendil-works/pi-coding-agent` 0.84.1 printed `ok - real Pi SDK 0.84.1 delivers a custom message to the provider as user text carrying only content, so the captain outcome's typed envelope is what reaches the model`. + The guard passes a typed captain outcome and a plain rendered routine note through Pi's exported `convertToLlm`, proves that `customType` and `display` are not model-visible identity, and classifies the resulting provider text with `bin/fm-operational-input.sh`. Scope of this evidence: the installed signed `pi` CLI (0.82.0 at verification time) is a compiled binary whose bundled SDK is not importable from Node, so the importable npm package is the only surface the guard and the typecheck can pin. The extension executes inside the signed CLI's own runtime, so a CLI upgrade can drift ahead of the pinned npm surface; refresh this record after every Pi upgrade by re-running the live guard, picker regression, and strict typecheck above (point `FM_PI_PACKAGE_DIR` at a matching npm install when one exists) and by watching the branch's own fallback line - every branch failure degrades to the pre-branch wake-to-main path by construction, which `tests/fm-pi-branch-extension.test.sh` holds with a broken generator and the live guard holds with the real SDK. diff --git a/tests/fm-pi-branch-extension.test.sh b/tests/fm-pi-branch-extension.test.sh index e4ac23e5eee..bf95b4587db 100644 --- a/tests/fm-pi-branch-extension.test.sh +++ b/tests/fm-pi-branch-extension.test.sh @@ -789,6 +789,55 @@ EOF pass "a captain outcome reaches main's model as typed, self-describing input while routine notes stay plain" } +test_captain_outcome_encoding_failure_delivers_plain_instruction() { + local repo home out status + repo="$TMP_ROOT/encoding-fallback-root" + home="$TMP_ROOT/encoding-fallback-home" + mkdir -p "$home/state" "$home/config" + install_pi_branch_extension_fixture "$repo" + PLUGIN="$repo/.pi/extensions/fm-branch-supervision.ts" FM_HOME="$home" FM_ROOT_OVERRIDE="$ROOT" \ + FM_OPERATIONAL_INPUT_SCRIPT="$repo/bin/missing-operational-input" \ + DRIVER_PRELUDE="$DRIVER_PRELUDE" node --input-type=module > "$TMP_ROOT/node-output" 2>&1 <<'EOF' +const prelude = process.env.DRIVER_PRELUDE; +await eval(`(async () => { ${prelude}; globalThis.__t = { dispatch, settle, sentToMain }; })()`); +const { dispatch, settle, sentToMain } = globalThis.__t; + +if (!dispatch("signal: encoding fallback probe").accepted) { + throw new Error("branch did not accept the encoding-fallback wake"); +} +await settle(() => (globalThis.__fmPrompts ?? []).length === 1, "encoding-fallback branch prompt"); +const session = globalThis.__fmSessions[0]; +const report = session.options.customTools.find((tool) => tool.name === "fm_branch_report"); +const result = await report.execute( + "encoding-fallback", + { task: "task-fallback", verdict: "captain", summary: "PR https://example.com/pr/fallback is ready" }, + undefined, + undefined, + {}, +); +if (result.isError) throw new Error(`fallback report failed: ${JSON.stringify(result)}`); +if (sentToMain.length !== 1) throw new Error(`fallback delivered ${sentToMain.length} notes instead of one`); +const delivered = sentToMain[0]; +if (delivered.message.display !== false) throw new Error("fallback captain note became visible"); +if (delivered.options.triggerTurn !== true || delivered.options.deliverAs !== "followUp") { + throw new Error(`fallback changed turn delivery: ${JSON.stringify(delivered.options)}`); +} +if (delivered.message.content.includes("FIRSTMATE_OP:")) { + throw new Error(`fallback unexpectedly carried an envelope: ${delivered.message.content}`); +} +if (!delivered.message.content.includes("Relay only this outcome") || + !delivered.message.content.includes("Do not restate or repeat any earlier answer") || + !delivered.message.content.includes("task-fallback: PR https://example.com/pr/fallback is ready")) { + throw new Error(`fallback lost its instruction or outcome: ${delivered.message.content}`); +} +process.exit(0); +EOF + status=$? + out=$(cat "$TMP_ROOT/node-output") + expect_code 0 "$status" "captain outcome encoding failure must degrade to plain instructed delivery: $out" + pass "a broken operational encoder still delivers one invisible instructed captain outcome as a follow-up" +} + test_branch_cache_key_is_per_home_stable() { local repo home_a home_b key_a1 key_a2 key_b repo="$TMP_ROOT/cache-key-root" @@ -2819,6 +2868,7 @@ JS test_outcomes_tool_uses_stock_execution_and_export_consumers test_real_pi_picker_primitives_stay_bounded_and_searchable test_branch_dispatch_two_stage_filter_and_prefix_contract +test_captain_outcome_encoding_failure_delivers_plain_instruction test_branch_dispatch_classifies_main_only_rows_and_writes_the_eligible_snapshot test_branch_cache_key_is_per_home_stable test_branch_default_on_heartbeat_afk_and_fallback