diff --git a/bin/fm-public-followup.sh b/bin/fm-public-followup.sh index a7c25cd18dd..1e9cf8b1f86 100755 --- a/bin/fm-public-followup.sh +++ b/bin/fm-public-followup.sh @@ -12,6 +12,8 @@ # state/x-context/ the private full request context (fm-x-lib.sh). # bin/fm-x-reply.sh posting to the relay, thread splitting, dry run. # bin/fm-public-followup-lib.sh the activation gate and private transport. +# bin/fm-on.sh the SSH route to a REMOTE secondmate home, whose +# state no local path can reach. # This script composes them; it never restates their contracts or schemas. # # ZERO OVERHEAD FOR HOMES THAT DO NOT USE THE RELAY: every subcommand gates @@ -105,7 +107,12 @@ # fm-public-followup.sh retire --reason "" [--force] # The only close. Drops the registration after recording --reason. # --force is the explicit discard-approved escape hatch for an unresolved -# or missing obligation. --reason is required. +# or missing obligation. --reason is required. --force never covers +# clearing the bound legacy X link: a loop whose link is still verifiably +# in place is retained for reconciliation either way. When the bound work +# lives in a REMOTE secondmate home, that clear runs over the route's SSH +# transport, and a remote that never confirms it is reported as unknown +# completion to reconcile on that host, not as a definite failure. # # Requires jq and a compatible tasks-axi for registration, briefs, # reconciliation, delivery, cleanup guards, and retirement; only `active` @@ -698,11 +705,47 @@ public_followup_secondmate_home() { printf '%s\n' "$home" } +# public_followup_route_is_remote : 0 when data/secondmates.md +# holds a genuine REMOTE route for that id. The registry is the route authority +# here for the same reason fm-on.sh and fm-send.sh treat it as one: a remote home +# has no local path, so nothing on this disk can answer the question. Resolving +# it live also means a registration written before this check (they all record an +# empty work_home_path for a remote route) still retires. +public_followup_route_is_remote() { + local id=$1 remote + fm_pf_home_id_valid "secondmate:$id" || return 1 + [ -f "$DATA/secondmates.md" ] && [ ! -L "$DATA/secondmates.md" ] || return 1 + remote=$(secondmate_registry_field "$DATA/secondmates.md" "$id" remote 2>/dev/null) || return 1 + [ "$remote" = 1 ] +} + +# clear_public_followup_link_remote : +# clear the bound legacy X link inside a REMOTE secondmate home over that route's transport, +# because the link lives in the remote home's state and no local path reaches it. +# fm-on.sh returns ssh's status unchanged, so 255 is the established "delivered +# but completion unknown" status this codebase already reconciles rather than +# reads as done or refused (bin/fm-on.sh, bin/fm-remote-readiness-lib.sh, +# bin/fm-teardown.sh). It is passed through so a caller can say the remote never +# confirmed instead of claiming the clear definitely failed. The remote clear +# is guarded by the registration's Relay request identity and remains idempotent +# when the target has no link, so a reconciling retry is safe. +clear_public_followup_link_remote() { + local id=$1 work_id=$2 request_id=$3 rc=0 + "$FM_ROOT/bin/fm-on.sh" "$id" fm-x-followup.sh --clear "$work_id" \ + --expect-request "$request_id" /dev/null || rc=$? + [ "$rc" -ne 255 ] || return 255 + [ "$rc" -eq 0 ] || return 1 + return 0 +} + +# Returns 0 when the link is cleared, 255 when a remote home never confirmed the +# clear (completion unknown), and 1 for any other refusal. clear_public_followup_link() { - local id=$1 work_home work_home_path work_id home state rc + local id=$1 work_home work_home_path work_id request_id home state rc public_followup_registration_valid "$id" || return 1 work_home=$(fm_pf_registry_get "$STATE" "$id" work_home) work_id=$(fm_pf_registry_get "$STATE" "$id" work_id) + request_id=$(fm_pf_registry_get "$STATE" "$id" request_id) [ -n "$work_home" ] && [ -n "$work_id" ] || return 1 case "$work_home" in main) @@ -710,6 +753,13 @@ clear_public_followup_link() { state=$STATE ;; secondmate:*) + # A remote route is decided from the registry BEFORE any local path is + # consulted: the recorded remote home path is meaningful only on its own + # host, so a same-named local directory must never stand in for it. + if public_followup_route_is_remote "${work_home#secondmate:}"; then + clear_public_followup_link_remote "${work_home#secondmate:}" "$work_id" "$request_id" + return $? + fi work_home_path=$(fm_pf_registry_get "$STATE" "$id" work_home_path) case "$work_home_path" in /*) ;; *) return 1 ;; esac case "$work_home_path" in *$'\n'*|*$'\r'*) return 1 ;; esac @@ -734,6 +784,17 @@ clear_public_followup_link() { "$FM_ROOT/bin/fm-x-followup.sh" --clear "$work_id" >/dev/null } +# pf_link_clear_note : the qualifier appended to a refusal when a bound +# legacy X link is still in place. Empty for every local refusal, so those +# messages are unchanged. A remote clear returns fm-on.sh's pass-through ssh +# status, where 255 means the remote home never confirmed the clear: completion +# is unknown and belongs to that host's reconciliation, never a definite failure +# and never a silent success. +pf_link_clear_note() { + [ "$1" -eq 255 ] || return 0 + printf ' The remote home never confirmed the clear, so reconcile it on that host rather than assuming nothing changed.' +} + public_followup_legacy_link_status() { local payload=$1 relations work_home work_id home meta if ! printf '%s' "$payload" | jq -e ' @@ -833,7 +894,7 @@ cmd_deliver() { || die "this home has not opted into the myfirstmate relay, so it cannot post a public reply" 1 require_tools - local payload delivery attempt request platform text tmp_text hash chunks rc receipt receipt_fields receipt_dry_run link_status + local payload delivery attempt request platform text tmp_text hash chunks rc receipt receipt_fields receipt_dry_run link_status link_rc local loop_retained=0 payload=$(obligation_json "$id") || die "could not read the backlog through tasks-axi" 1 [ -n "$payload" ] || die "no public-followup obligation '$id' in this home's backlog" 1 @@ -847,8 +908,10 @@ cmd_deliver() { case "$delivery" in posted|waived) if public_followup_registration_valid "$id"; then - if ! clear_public_followup_link "$id"; then - die "obligation '$id' is already $delivery, but its legacy X link could not be cleared; the registration was retained for reconciliation" 1 + link_rc=0 + clear_public_followup_link "$id" || link_rc=$? + if [ "$link_rc" -ne 0 ]; then + die "obligation '$id' is already $delivery, but its legacy X link could not be cleared; the registration was retained for reconciliation$(pf_link_clear_note "$link_rc")" 1 fi else link_status=1 @@ -939,8 +1002,10 @@ EOF die "dry-run for '$id' did not post; recorded as retryable and left the obligation open" 1 fi if record_posted "$id" "$attempt" "$request" "$platform" "$chunks"; then - if ! clear_public_followup_link "$id"; then - die "the public reply for '$id' POSTED and its receipt was recorded, but its legacy X link could not be cleared; the registration was retained for reconciliation" 1 + link_rc=0 + clear_public_followup_link "$id" || link_rc=$? + if [ "$link_rc" -ne 0 ]; then + die "the public reply for '$id' POSTED and its receipt was recorded, but its legacy X link could not be cleared; the registration was retained for reconciliation$(pf_link_clear_note "$link_rc")" 1 fi if mark_loop_delivered "$id"; then loop_retained=1; fi printf 'delivered %s request=%s platform=%s chunks=%s\n' "$id" "$request" "$platform" "$chunks" @@ -969,7 +1034,7 @@ EOF # --- subcommand: record-posted --------------------------------------------- cmd_record_posted() { - local id=${1:-} attempt='' chunks='' + local id=${1:-} attempt='' chunks='' link_rc [ -n "$id" ] || { usage; exit 2; } shift while [ "$#" -gt 0 ]; do @@ -997,8 +1062,10 @@ cmd_record_posted() { record_posted "$id" "$attempt" "$request" "$platform" "$chunks" \ || die "tasks-axi refused the receipt for '$id' attempt $attempt; the recorded attempt must match exactly" 1 - if ! clear_public_followup_link "$id"; then - die "the receipt for '$id' was recorded, but its legacy X link could not be cleared; the registration was retained for reconciliation" 1 + link_rc=0 + clear_public_followup_link "$id" || link_rc=$? + if [ "$link_rc" -ne 0 ]; then + die "the receipt for '$id' was recorded, but its legacy X link could not be cleared; the registration was retained for reconciliation$(pf_link_clear_note "$link_rc")" 1 fi if mark_loop_delivered "$id"; then loop_retained=1; fi printf 'recorded %s attempt=%s request=%s\n' "$id" "$attempt" "$request" @@ -1251,7 +1318,7 @@ cmd_rechain() { # --- subcommand: retire ----------------------------------------------------- cmd_retire() { - local id=${1:-} force=0 reason='' payload delivery task_state registry_file retired_dir retired_at + local id=${1:-} force=0 reason='' payload delivery task_state registry_file retired_dir retired_at link_rc local retirement_rc=0 [ -n "$id" ] || { usage; exit 2; } shift @@ -1284,8 +1351,10 @@ cmd_retire() { ;; esac fi - if ! clear_public_followup_link "$id"; then - die "could not clear the legacy X link for '$id'; its registration was retained for reconciliation" 1 + link_rc=0 + clear_public_followup_link "$id" || link_rc=$? + if [ "$link_rc" -ne 0 ]; then + die "could not clear the legacy X link for '$id'; its registration was retained for reconciliation$(pf_link_clear_note "$link_rc")" 1 fi retired_dir=$(fm_pf_retired_dir "$STATE") retired_at=$(now_rfc3339) diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index 0b958895551..a9ccddcb02b 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -938,10 +938,13 @@ _fm_lock_acquire_wait_handoff() { # # fm_lock_acquire_wait_bounded # -# Presentation-only acquire variant. It preserves the ordinary wait/reclaim -# behavior until fm-timeout-lib.sh's hard deadline, returns 124 when a live -# holder still owns the lock, and leaves FM_LOCK_HELD_PID naming that holder. -# Mutation-critical callers continue to use fm_lock_acquire_wait. +# Bounded acquire variant. It preserves the ordinary wait/reclaim behavior +# until fm-timeout-lib.sh's hard deadline, returns 124 when a live holder still +# owns the lock, and leaves FM_LOCK_HELD_PID naming that holder. +# Use it where a caller must refuse rather than block: wake presentation, and +# the guarded remote link clear, whose whole contract is to return a +# reconciliation refusal instead of wedging an unattended close. +# Mutation-critical callers that can safely block keep fm_lock_acquire_wait. fm_lock_acquire_wait_bounded() { local lockdir=$1 seconds=$2 caller_pid rc owner_pid case "$seconds" in ''|*[!0-9]*|0) return 2 ;; esac diff --git a/bin/fm-x-followup.sh b/bin/fm-x-followup.sh index e19c8c3a19d..b847e7b059a 100755 --- a/bin/fm-x-followup.sh +++ b/bin/fm-x-followup.sh @@ -18,9 +18,9 @@ # pruned) # # Clear a legacy link without posting: -# fm-x-followup.sh --clear +# fm-x-followup.sh --clear [--expect-request ] # idempotently removes only the X follow-up metadata for a typed terminal -# outcome. +# outcome. With --expect-request, a present link must match that request. # # Post (after composing the reply to a file or stdin): # fm-x-followup.sh [--image ] [--final] --text-file @@ -72,13 +72,13 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-wake-lib.sh" usage() { - echo "usage: fm-x-followup.sh --check | --clear | [--image ] [--final] --text-file | [--image ] [--final] -" >&2 + echo "usage: fm-x-followup.sh --check | --clear [--expect-request ] | [--image ] [--final] --text-file | [--image ] [--final] -" >&2 } help() { cat <<'EOF' usage: fm-x-followup.sh --check - fm-x-followup.sh --clear + fm-x-followup.sh --clear [--expect-request ] fm-x-followup.sh [--image ] [--final] --text-file fm-x-followup.sh [--image ] [--final] - @@ -88,6 +88,8 @@ X-mode-linked task and manage the link's follow-up counter. Options: --check Print the request_id when a follow-up is due. --clear Clear only the X follow-up link; never post. + --expect-request + With --clear, require a present link to match this request. --image Attach one local image file; threaded replies attach it to the opener tweet or message. --final Clear the link after this post regardless of the remaining count. --text-file @@ -117,10 +119,19 @@ case "${1:-}" in esac FINAL=0 +EXPECT_REQUEST_SET=0 +EXPECT_REQUEST= if [ "${1:-}" = --clear ]; then MODE=clear ID=${2:-} - if [ -z "$ID" ] || [ "$#" -gt 2 ]; then usage; exit 2; fi + if [ "$#" -eq 4 ] && [ "${3:-}" = --expect-request ]; then + EXPECT_REQUEST_SET=1 + EXPECT_REQUEST=${4-} + elif [ "$#" -ne 2 ]; then + usage + exit 2 + fi + if [ -z "$ID" ]; then usage; exit 2; fi elif [ "${1:-}" = --check ]; then MODE=check ID=${2:-} @@ -162,8 +173,13 @@ if [ -e "$META" ] || [ -L "$META" ]; then || { echo "fm-x-followup: unsafe task record in state/$ID.meta" >&2; exit 1; } fi if [ "$MODE" = clear ]; then - fmx_meta_link_clear "$META" \ - || { echo "fm-x-followup: could not clear the link in state/$ID.meta" >&2; exit 1; } + if [ "$EXPECT_REQUEST_SET" -eq 1 ]; then + fmx_meta_link_clear "$META" "$EXPECT_REQUEST" \ + || { echo "fm-x-followup: could not clear the link in state/$ID.meta" >&2; exit 1; } + else + fmx_meta_link_clear "$META" \ + || { echo "fm-x-followup: could not clear the link in state/$ID.meta" >&2; exit 1; } + fi printf '%s\n' "$ID" exit 0 fi diff --git a/bin/fm-x-lib.sh b/bin/fm-x-lib.sh index e6976664350..f50ddce781d 100644 --- a/bin/fm-x-lib.sh +++ b/bin/fm-x-lib.sh @@ -976,18 +976,68 @@ fmx_meta_followups_set() { fm_lock_release "$lock" } -# fmx_meta_link_clear : atomically remove the x_request/x_request_ts/ -# x_followups and reply-platform lines while preserving every other meta line. Idempotent: -# succeeds whether or not a link is present, and is a no-op when is -# missing. +# fmx_meta_link_clear [expected-request]: atomically remove the +# x_request/x_request_ts/x_followups and reply-platform lines while preserving +# every other meta line. With expected-request, a present link is cleared only +# when its request identity matches, and absence succeeds only when the +# authorized parent directory can be inspected safely. That guarded mode also +# bounds its lock wait (FMX_LINK_CLEAR_LOCK_TIMEOUT, default 10 seconds) so an +# unattended remote clear refuses instead of hanging. Unguarded calls remain +# idempotent when is missing and keep the ordinary unbounded wait. fmx_meta_link_clear() { - local meta=$1 tmp lock + local meta=$1 expected_set=0 expected='' tmp lock line rid='' link_present=0 parent + local lock_timeout + if [ "$#" -ge 2 ]; then + expected_set=1 + expected=$2 + parent=${meta%/*} + [ "$parent" != "$meta" ] || parent=. + [ -d "$parent" ] && [ ! -L "$parent" ] && [ -r "$parent" ] \ + && [ -x "$parent" ] || return 1 + fm_backlog_record_parent_authorized "$meta" "task record" "$STATE" || return 1 + fi [ ! -L "$meta" ] || return 1 [ -f "$meta" ] || return 0 + if [ "$expected_set" -eq 1 ]; then + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + x_request=*) link_present=1; rid=${line#*=} ;; + esac + done < "$meta" || return 1 + [ "$link_present" -eq 1 ] || return 0 + [ -n "$expected" ] && [ -n "$rid" ] && [ "$rid" = "$expected" ] || return 1 + [ -w "$parent" ] || return 1 + fi lock=$(fm_meta_lock_path "$meta") || return 1 - fm_lock_acquire_wait "$lock" + if [ "$expected_set" -eq 1 ]; then + # A guarded clear runs unattended over the secondmate transport, so it must + # refuse rather than wedge. The parent's writability can flip between the + # check above and lock creation, and the ordinary unbounded wait would then + # retry forever instead of returning the reconciliation refusal this guard + # exists to produce. A bounded acquire turns that race, and a live holder, + # into a refusal. Unguarded local callers keep the ordinary wait unchanged. + lock_timeout=${FMX_LINK_CLEAR_LOCK_TIMEOUT:-10} + case "$lock_timeout" in ''|*[!0-9]*|0) lock_timeout=10 ;; esac + fm_lock_acquire_wait_bounded "$lock" "$lock_timeout" || return 1 + else + fm_lock_acquire_wait "$lock" + fi [ ! -L "$meta" ] || { fm_lock_release "$lock"; return 1; } [ -f "$meta" ] || { fm_lock_release "$lock"; return 0; } + if [ "$expected_set" -eq 1 ]; then + link_present=0 + rid= + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + x_request=*) link_present=1; rid=${line#*=} ;; + esac + done < "$meta" || { fm_lock_release "$lock"; return 1; } + [ "$link_present" -eq 0 ] || { + [ -n "$expected" ] && [ -n "$rid" ] && [ "$rid" = "$expected" ] \ + || { fm_lock_release "$lock"; return 1; } + } + [ "$link_present" -eq 1 ] || { fm_lock_release "$lock"; return 0; } + fi tmp=$(fmx_meta_tmp "$meta") || { fm_lock_release "$lock"; return 1; } if ! { grep -vE '^x_request=|^x_request_ts=|^x_followups=|^x_platform=|^x_reply_max_chars=' "$meta" || true; } > "$tmp"; then rm -f "$tmp"; fm_lock_release "$lock"; return 1 diff --git a/docs/architecture.md b/docs/architecture.md index 027efe769f1..41f1745dd82 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -318,7 +318,8 @@ Actionable reversible requests run through firstmate's normal intake, backlog, d Work that completes in the answering turn gets one outcome reply. Work that spawns a longer-running task gets an acknowledgement reply first; `bin/fm-x-link.sh` records `x_request=`, `x_request_ts=`, `x_followups=0`, and optional reply-platform context in that task's `state/.meta`, while durable per-request context preserves the original platform and budget independently of task links and inbox cleanup. That link therefore reaches only work whose task record lives in the answering home; work routed to a secondmate is bound instead by a typed promised-final commitment registered with `--work-home secondmate:`, and `bin/fm-x-link.sh` refuses a non-local task with that path named rather than leaving the public promise unbound. -Later milestone wakes use `bin/fm-x-followup.sh` to post up to three public-safe follow-ups through the relay's `connector/followup` endpoint, ending with a `--final` one for ordinary Relay-linked work. A typed promised-final commitment owns its terminal reply through `bin/fm-public-followup.sh`; after its receipt is validated, `bin/fm-x-followup.sh --clear ` removes any legacy link without posting another reply. +Later milestone wakes use `bin/fm-x-followup.sh` to post up to three public-safe follow-ups through the relay's `connector/followup` endpoint, ending with a `--final` one for ordinary Relay-linked work. +A typed promised-final commitment owns its terminal reply through `bin/fm-public-followup.sh`; after its receipt is validated, that owner asks the bound work home to remove any legacy link without posting another reply, routing a REMOTE secondmate clear through its SSH transport with the registration's Relay request identity as the mutation guard. The [Relay configuration reference](configuration.md#relay-env) owns the exact context retention, platform-resolution, and fail-safe posting contract. If recovery relinks the same relay request onto a successor task, `fm-x-link.sh --carry-count --carry-ts --carry-platform --carry-max ` preserves the consumed follow-up count, original 7-day window, and reply split budget instead of granting a fresh local budget or falling back to the wrong platform. The follow-up helper forwards `--image ` to the same reply client when a follow-up needs an image. diff --git a/docs/configuration.md b/docs/configuration.md index 17f91b3b61c..0bf20a3407b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -580,6 +580,8 @@ Run `bin/fm-public-followup.sh --help` for the exact subcommands and flags. Registration is what creates this home's private transport under `state/public-followup/` (mode 0700): `registry/` for the bounded private binding of each open public loop (the record survives delivery, stamped `state=delivered`, and is removed only by `retire`), `events/` for typed terminal results awaiting reconciliation, `consumed/` for the accepted-event ledger, `rejected/` for refusals kept with a one-line reason, `retired/` for the mode-0600 reason-and-time receipt written before removal, and `surfaced` for the poll's last-surfaced signature. The home that owns the commitment also owns the outward post, because only it holds the relay consent, the request context, and the opaque thread binding. Work routed elsewhere reports a typed terminal result with `bin/fm-public-followup-emit.sh` and never looks for the thread; that emitter refuses to write into a home with no registration for the named obligation. +When that work lives in a REMOTE secondmate home, delivery clears its bound legacy link after validating the public receipt, while retirement clears the link before closing the loop, and both clears run over that route's SSH transport. +Readable remote state that proves no link exists succeeds without a write, while a present link is cleared only when its Relay request identity matches the registration and the state is writable; an identity mismatch, unreadable or unsafe state, an unavailable write or lock, an older remote copy, or a host that never confirms the clear leaves the loop retained for reconciliation. A terminal event's id is derived from its identity tuple, so a duplicate report, a retry, or a replay after restart resolves to the same event and changes nothing. Activation is the same `.env` `FMX_PAIRING_TOKEN` contract as the rest of Relay, with no second flag. diff --git a/docs/verification/public-followup.md b/docs/verification/public-followup.md index 6a13b2d09a3..29cfb428638 100644 --- a/docs/verification/public-followup.md +++ b/docs/verification/public-followup.md @@ -2,21 +2,23 @@ Audience: maintainer verification. -This record supports four active guarantees for promised public replies made through the myfirstmate relay: +This record supports five active guarantees for promised public replies made through the myfirstmate relay: 1. A promised final reply survives compaction and restart, reconciles from disk alone, and lands in the original thread exactly once. 2. A home that never opted into the relay pays nothing for any of it. 3. Delivering a final does not close the public loop: the registration is retained as `state=delivered` until `retire --reason`, session start surfaces an `open-loop` line, and `rechain` can bind follow-on work to the same thread. 4. A first registration with no registry lock already held succeeds under stock macOS Bash 3.2 with `set -u`. +5. A public loop whose work lives in a REMOTE secondmate home retires when readable remote state proves no link exists, or after readable and writable remote state clears the matching bound legacy Relay link; unreadable state, a non-writable matching link, an identity mismatch, a metadata lock it cannot acquire within its bound, or unconfirmed completion retains the loop instead of hanging, and `--force` still covers only the unresolved obligation. [`docs/configuration.md`](../configuration.md#promised-public-replies-statepublic-followup) owns the operator-facing contract, [`docs/architecture.md`](../architecture.md#optional-relay) owns the mechanism boundary, and `tasks-axi public-followup --help` owns the typed obligation schema. Task chronology and delivery evidence stay outside this record. ## Environment -Recorded 2026-08-21 on Darwin 25.5.0 (arm64) with GNU bash 5.3.9, tasks-axi 0.2.5, jq 1.8.1, and ShellCheck 0.11.0 (the version `bin/fm-lint.sh` pins). +Recorded 2026-09-01 on Darwin 25.5.0 (arm64) with GNU bash 5.3.9, tasks-axi 0.2.5, jq 1.8.1, and ShellCheck 0.11.0 (the version `bin/fm-lint.sh` pins). The stock macOS compatibility lane additionally runs the focused first-registration regression with `/bin/bash` 3.2.57 and a real `tasks-axi` installation. The relay is a fakebin `curl` in every case, so no public post is ever made; `tasks-axi` and `jq` are the real tools, because stubbing the obligation state machine would verify nothing. +The remote-route cases fake only the SSH binary at the `FM_SSH_BIN` process seam and then run the real tracked `fm-remote-entrypoint.sh` against a local checkout standing in for the remote one, so the clear that has to reach the remote home actually runs there; no host and no network are involved. ## Restart end-to-end and regressions @@ -78,6 +80,14 @@ ok - brief fails explicitly when typed deliverable keys are unavailable ok - pre-change registrations are open loops and un-rechainable, never a crash ok - teardown reports an unreconciled legacy Relay link ok - secondmate promotion matches teardown parent resolution +ok - a public loop bound to a remote secondmate home delivers and retires +ok - --force still covers only the unresolved obligation, not the link clear +ok - retire fails closed when a remote route is reassigned +ok - retire fails closed when remote state is unreadable +ok - retire fails closed when remote state is non-writable +ok - retire accepts link absence in non-writable remote state +ok - the guarded remote clear refuses a lock it cannot acquire instead of hanging +ok - an unconfirmed remote clear is unknown completion, never a silent close ``` The restart case is the end-to-end proof of guarantee 1. @@ -91,7 +101,19 @@ The concurrency and interrupted-bind cases verify that one delivered source cann A pre-change on-disk record (no `state=`, no `request_context_b64`) is an open loop and un-rechainable rather than a crash. The stock macOS Bash lane in [`.github/workflows/ci.yml`](../../.github/workflows/ci.yml) sets `FM_TEST_ONLY=test_first_register_succeeds_with_empty_lock_list_under_bash32` and runs `tests/fm-public-followup.test.sh` through real `/bin/bash` 3.2, proving the first `register` path is safe when its registry lock list starts empty. -The existing Relay mention suite (`tests/fm-x-mode.test.sh`) is unchanged by this work. +The eight remote-route cases are the proof of guarantee 5. +A remote secondmate home exists only on its own machine, so its registration records no local path, and every close that must first clear the bound legacy Relay link had nothing local to act on. +The first case pins that empty recorded path so it cannot go vacuous, then drives `deliver` and `retire` end to end and asserts the matching link inside the remote home is actually gone and the retirement receipt is written. +The second case shows `--force` still governs only the unresolved-obligation refusal: a plain `retire` of an unresolved remote loop is still refused with the remote link untouched, while a forced one closes and clears it. +The reassignment case replaces a delivered loop's route with a remote home whose reused work ID carries another Relay request and asserts that retirement retains the registration and leaves the replacement link untouched. +The unreadable-state case makes the remote state directory non-searchable while it still contains a matching link and proves that an unconfirmable path fails closed without mutation. +The two non-writable-state cases prove that a matching link refuses before lock acquisition because mutation is impossible, while a confirmed absent link succeeds because no mutation is needed. +The unacquirable-lock case is the proof that the guarded clear refuses rather than wedges. +It leaves the remote state directory WRITABLE, so the refusal can only come from the bounded lock wait and never from the writability precondition, and holds the metadata lock with a genuinely live process so the lock can never be reclaimed as stale. +The writability precondition narrows the wedge window but cannot close it, because the parent can turn non-writable between that check and lock creation and a live holder is indistinguishable from it at the acquire; the ordinary unbounded wait retries forever, so before the bounded acquire this path hung with nothing reported instead of returning the reconciliation refusal. +The case asserts the refusal, the retained registration, the absent receipt, the untouched remote link, and that the call returns at all, which is the observable difference from a wait that never ends. +The final case makes the transport unreachable and asserts the close is refused with the registration retained, the remote link untouched, and unknown completion named rather than reported as a definite failure. +A remote home running an older Firstmate copy does not recognize the guarded clear flag and therefore fails closed through the same retained-for-reconciliation message; operators must update that home before retrying, and there is deliberately no unguarded fallback. ## Relay-disabled zero overhead diff --git a/tests/fm-public-followup.test.sh b/tests/fm-public-followup.test.sh index f4c9aadac0f..d9cea013167 100755 --- a/tests/fm-public-followup.test.sh +++ b/tests/fm-public-followup.test.sh @@ -15,6 +15,8 @@ set -u # shellcheck source=tests/lib.sh # shellcheck disable=SC1091 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=bin/fm-timeout-lib.sh +. "$ROOT/bin/fm-timeout-lib.sh" PF="$ROOT/bin/fm-public-followup.sh" EMIT="$ROOT/bin/fm-public-followup-emit.sh" @@ -24,6 +26,27 @@ PROMOTE="$ROOT/bin/fm-promote.sh" SESSION_START="$ROOT/bin/fm-session-start.sh" TMP_ROOT=$(fm_test_tmproot fm-public-followup) PF_TEST_NOW=1787539200 +PF_TEST_LOCK_HOLDER= + +# The remote-route cases drive the real remote job worker, which outlives the +# command that staged its job. Stop it before the shared fixture cleanup runs, +# and keep that cleanup (tests/lib.sh owns it) rather than replacing the trap. +pf_test_cleanup() { + local pid_file="${REMOTE_FIXTURE_JOBS:-$TMP_ROOT/remote-jobs}/worker.pid" pid + if [ -n "$PF_TEST_LOCK_HOLDER" ]; then + kill "$PF_TEST_LOCK_HOLDER" 2>/dev/null || true + wait "$PF_TEST_LOCK_HOLDER" 2>/dev/null || true + PF_TEST_LOCK_HOLDER= + fi + if [ -f "$pid_file" ]; then + pid=$(cat "$pid_file" 2>/dev/null) || pid= + [ -z "$pid" ] || kill "$pid" 2>/dev/null || true + fi + fm_test_cleanup +} +trap pf_test_cleanup EXIT +trap 'pf_test_cleanup; exit 130' INT +trap 'pf_test_cleanup; exit 143' TERM command -v jq >/dev/null 2>&1 || { echo "skip: jq not found"; exit 0; } command -v tasks-axi >/dev/null 2>&1 || { echo "skip: tasks-axi not found"; exit 0; } @@ -2272,6 +2295,395 @@ test_secondmate_promotion_uses_teardown_parent_resolution() { pass "secondmate promotion matches teardown parent resolution" } +# --- remote secondmate work homes --------------------------------------------- +# +# A REMOTE secondmate route records no local path for its home, because the home +# only exists on the other machine. Registration therefore stores an empty +# work_home_path, and every close that must first clear the bound legacy X link +# has to reach that home over the route's SSH transport instead. +# +# The transport is faked at the FM_SSH_BIN process seam and then runs the REAL +# tracked remote entrypoint against a local "remote" checkout, so the clear that +# has to happen actually happens: no live host, no network, and no assumption +# baked into a stub about what the far side would have done. + +REMOTE_FIXTURE_ROOT= +REMOTE_FIXTURE_SSH= +REMOTE_FIXTURE_JOBS= + +# remote_fixture_prepare: build the shared remote checkout and fake ssh once. +# The remote root is a real git repo holding the real bin/, because both fm-on.sh +# and the entrypoint refuse anything that is not a genuine tracked executable. +remote_fixture_prepare() { + local fakebin + [ -z "$REMOTE_FIXTURE_ROOT" ] || return 0 + # TMPDIR on macOS carries a trailing slash, and the route validation rejects an + # empty path component, so physicalize both fixture paths before registering. + REMOTE_FIXTURE_ROOT="$TMP_ROOT/remote-root" + mkdir -p "$REMOTE_FIXTURE_ROOT/bin/backends" + REMOTE_FIXTURE_ROOT=$(cd "$REMOTE_FIXTURE_ROOT" && pwd -P) + mkdir -p "$TMP_ROOT/remote-jobs" + REMOTE_FIXTURE_JOBS=$(cd "$TMP_ROOT/remote-jobs" && pwd -P) + cp "$ROOT"/bin/fm-*.sh "$REMOTE_FIXTURE_ROOT/bin/" + cp "$ROOT"/bin/backends/*.sh "$REMOTE_FIXTURE_ROOT/bin/backends/" + chmod +x "$REMOTE_FIXTURE_ROOT/bin"/*.sh + printf 'fixture\n' > "$REMOTE_FIXTURE_ROOT/AGENTS.md" + git -C "$REMOTE_FIXTURE_ROOT" init -q -b main + git -C "$REMOTE_FIXTURE_ROOT" config user.email test@example.com + git -C "$REMOTE_FIXTURE_ROOT" config user.name Test + git -C "$REMOTE_FIXTURE_ROOT" add AGENTS.md bin + git -C "$REMOTE_FIXTURE_ROOT" commit -qm 'tracked remote fixture' + + fakebin=$(fm_fakebin "$TMP_ROOT/remote-transport") + cat > "$fakebin/fake-ssh" <<'SH' +#!/usr/bin/env bash +while [ "$#" -gt 0 ]; do + case "$1" in + -o) shift 2 ;; + --) shift; break ;; + *) exit 90 ;; + esac +done +host=$1 +entry=$2 +shift 2 +[ "$host" = remote-mac ] || exit 91 +[ "$entry" = fm-remote-entrypoint.sh ] || exit 92 +case "${FM_FAKE_SSH_MODE:-normal}" in + unreachable) exit 255 ;; + *) exec "$FM_FAKE_REMOTE_ENTRYPOINT" "$@" ;; +esac +SH + chmod +x "$fakebin/fake-ssh" + REMOTE_FIXTURE_SSH="$fakebin/fake-ssh" +} + +# make_remote_route : register a REMOTE secondmate route in +# and echo the path standing in for that secondmate's home on the far +# machine. The parent-side task record carries the same route fm-spawn writes. +# Callers run remote_fixture_prepare first, because this one is used in command +# substitution and a subshell cannot publish the shared fixture globals. +make_remote_route() { # + local home=$1 id=$2 remote_home + remote_home="$TMP_ROOT/$(basename "$home")-remote-$id" + mkdir -p "$remote_home/state" "$remote_home/data" + remote_home=$(cd "$remote_home" && pwd -P) + cat > "$home/data/secondmates.md" < + local home=$1 + shift + PATH="$home/fakebin:$PATH" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FAKE_CURL_LOG="${FAKE_CURL_LOG:-}" \ + FAKE_FOLLOWUP_CODE="${FAKE_FOLLOWUP_CODE:-200}" \ + FMX_NOW_OVERRIDE="${FMX_NOW_OVERRIDE:-$PF_TEST_NOW}" \ + FM_SSH_BIN="$REMOTE_FIXTURE_SSH" \ + FM_FAKE_SSH_MODE="${FM_FAKE_SSH_MODE:-normal}" \ + FM_FAKE_REMOTE_ENTRYPOINT="$REMOTE_FIXTURE_ROOT/bin/fm-remote-entrypoint.sh" \ + FM_REMOTE_JOB_PLATFORM_OVERRIDE=Linux \ + FM_REMOTE_JOB_STATE_ROOT="$REMOTE_FIXTURE_JOBS" \ + "$PF" "$@" +} + +run_pf_remote_timed() { # + local seconds=$1 home=$2 + shift 2 + fm_run_timed "$seconds" env \ + PATH="$home/fakebin:$PATH" FM_ROOT_OVERRIDE="$ROOT" FM_HOME="$home" \ + FM_STATE_OVERRIDE="$home/state" FAKE_CURL_LOG="${FAKE_CURL_LOG:-}" \ + FAKE_FOLLOWUP_CODE="${FAKE_FOLLOWUP_CODE:-200}" \ + FMX_NOW_OVERRIDE="${FMX_NOW_OVERRIDE:-$PF_TEST_NOW}" \ + FM_SSH_BIN="$REMOTE_FIXTURE_SSH" \ + FM_FAKE_SSH_MODE="${FM_FAKE_SSH_MODE:-normal}" \ + FM_FAKE_REMOTE_ENTRYPOINT="$REMOTE_FIXTURE_ROOT/bin/fm-remote-entrypoint.sh" \ + FM_REMOTE_JOB_PLATFORM_OVERRIDE=Linux \ + FM_REMOTE_JOB_STATE_ROOT="$REMOTE_FIXTURE_JOBS" \ + "$PF" "$@" +} + +# The reported failure: a public loop whose work lived in a REMOTE secondmate +# home could never be closed. Its registration carries no local path, so the +# legacy-link clear that every close runs first had nothing to act on and refused +# forever - leaving the promise permanently open and, on the delivery path, +# leaving a loop stuck at posted after the public reply had already landed. +test_remote_secondmate_loop_delivers_and_retires() { + local home remote log + remote_fixture_prepare + home=$(make_home remote-retire) + remote=$(make_remote_route "$home" mini-default) + log="$home/curl.log"; : > "$log" + seed_repro_commitment "$home" pf-remote-close req-remote-close secondmate:mini-default work-remote + fm_write_meta "$remote/state/work-remote.meta" \ + "x_request=req-remote-close" "x_request_ts=1700000000" "x_followups=1" + + # The trap condition, pinned so this case can never go vacuous: a remote route + # has no local home path to record, which is exactly what used to dead-end. + [ -z "$(sed -n 's/^work_home_path=//p' "$home/state/public-followup/registry/pf-remote-close")" ] \ + || fail "a remote work home must register with no local path" + + "$EMIT" --home "$home" --obligation pf-remote-close --relation rel-code \ + --source-home secondmate:mini-default --work-id work-remote --generation 1 \ + --outcome report-ready --deliverable report_path=data/work-remote/report.md \ + --outcome-text 'The remote lane finished its investigation.' >/dev/null \ + || fail "emit failed" + run_pf "$home" consume >/dev/null || fail "consume failed" + + FAKE_CURL_LOG="$log" run_pf_remote "$home" deliver pf-remote-close >/dev/null \ + || fail "delivery must not strand a remote-home loop after the public reply lands" + assert_no_grep 'x_request=' "$remote/state/work-remote.meta" \ + "delivery must clear the legacy X link inside the remote home" + [ "$(delivery_state "$home" pf-remote-close)" = posted ] \ + || fail "a delivered remote-home loop must reach posted" + + run_pf_remote "$home" retire pf-remote-close --reason "handed on by hand" >/dev/null \ + || fail "retire must be able to close a delivered remote-home loop" + assert_present "$home/state/public-followup/retired/pf-remote-close" \ + "retiring a remote-home loop must record its receipt" + assert_absent "$home/state/public-followup/registry/pf-remote-close" \ + "retiring a remote-home loop must drop its registration" + pass "a public loop bound to a remote secondmate home delivers and retires" +} + +# --force governs the unresolved-obligation refusal and nothing else. It never +# covered the legacy-link clear before this fix and must not start to now: a link +# still verifiably in place keeps the loop open on either setting. +test_remote_retire_force_semantics_unchanged() { + local home remote + remote_fixture_prepare + home=$(make_home remote-force) + remote=$(make_remote_route "$home" mini-default) + seed_repro_commitment "$home" pf-remote-open req-remote-open secondmate:mini-default work-open + seed_repro_commitment "$home" pf-remote-forced req-remote-forced secondmate:mini-default work-forced + fm_write_meta "$remote/state/work-open.meta" \ + "x_request=req-remote-open" "x_request_ts=1700000000" "x_followups=1" + fm_write_meta "$remote/state/work-forced.meta" \ + "x_request=req-remote-forced" "x_request_ts=1700000000" "x_followups=1" + + expect_failure "an unresolved remote loop must still refuse a plain retire" \ + run_pf_remote "$home" retire pf-remote-open --reason "not done yet" + assert_contains "$EXPECT_OUT" "hide an open public promise" \ + "the refusal must still be the unresolved-obligation one" + assert_present "$home/state/public-followup/registry/pf-remote-open" \ + "a refused retire must keep the registration" + assert_grep 'x_request=req-remote-open' "$remote/state/work-open.meta" \ + "a refused retire must not touch the remote home's link" + + run_pf_remote "$home" retire pf-remote-forced --reason "discarded" --force >/dev/null \ + || fail "--force must still discard an unresolved remote-home loop" + assert_present "$home/state/public-followup/retired/pf-remote-forced" \ + "a forced retire must record its receipt" + assert_no_grep 'x_request=' "$remote/state/work-forced.meta" \ + "a forced retire must still clear the remote home's link" + pass "--force still covers only the unresolved obligation, not the link clear" +} + +test_remote_retire_refuses_reassigned_route() { + local home original replacement log + remote_fixture_prepare + home=$(make_home remote-reassigned) + original=$(make_remote_route "$home" mate) + log="$home/curl.log"; : > "$log" + seed_repro_commitment "$home" pf-remote-reassigned req-remote-original secondmate:mate work-reused + fm_write_meta "$original/state/work-reused.meta" \ + "x_request=req-remote-original" "x_request_ts=1700000000" "x_followups=1" + "$EMIT" --home "$home" --obligation pf-remote-reassigned --relation rel-code \ + --source-home secondmate:mate --work-id work-reused --generation 1 \ + --outcome report-ready --deliverable report_path=data/work-reused/report.md \ + --outcome-text 'The original remote route finished its work.' >/dev/null || fail "emit failed" + run_pf "$home" consume >/dev/null || fail "consume failed" + FAKE_CURL_LOG="$log" run_pf_remote "$home" deliver pf-remote-reassigned >/dev/null \ + || fail "delivery through the original remote route must succeed" + + replacement="$TMP_ROOT/remote-replacement-mate" + mkdir -p "$replacement/state" "$replacement/data" + replacement=$(cd "$replacement" && pwd -P) + cat > "$home/data/secondmates.md" <&1) || rc=$? + chmod 700 "$remote/state" + [ "$rc" -ne 0 ] || fail "retire must refuse an unreadable remote state (unexpectedly succeeded)" + assert_contains "$EXPECT_OUT" "could not clear the legacy X link" \ + "an unreadable remote state must use the retained reconciliation refusal" + assert_present "$home/state/public-followup/registry/pf-remote-unreadable" \ + "an unreadable remote state must retain the registration" + assert_absent "$home/state/public-followup/retired/pf-remote-unreadable" \ + "an unreadable remote state must not write a retirement receipt" + assert_grep 'x_request=req-remote-unreadable' "$meta" \ + "an unreadable remote state must leave the link untouched" + pass "retire fails closed when remote state is unreadable" +} + +test_remote_retire_refuses_nonwritable_state() { + local home remote meta rc + remote_fixture_prepare + home=$(make_home remote-nonwritable) + remote=$(make_remote_route "$home" mate) + seed_repro_commitment "$home" pf-remote-nonwritable req-remote-nonwritable secondmate:mate work-nonwritable + meta="$remote/state/work-nonwritable.meta" + fm_write_meta "$meta" \ + "status=working" "x_request=req-remote-nonwritable" "x_request_ts=1700000000" "x_followups=1" + chmod 500 "$remote/state" + + rc=0 + EXPECT_OUT=$(run_pf_remote "$home" retire pf-remote-nonwritable --reason "cannot mutate" --force 2>&1) || rc=$? + chmod 700 "$remote/state" + [ "$rc" -ne 0 ] || fail "retire must refuse a non-writable remote state (unexpectedly succeeded)" + assert_contains "$EXPECT_OUT" "could not clear the legacy X link" \ + "a non-writable remote state must use the retained reconciliation refusal" + assert_present "$home/state/public-followup/registry/pf-remote-nonwritable" \ + "a non-writable remote state must retain the registration" + assert_absent "$home/state/public-followup/retired/pf-remote-nonwritable" \ + "a non-writable remote state must not write a retirement receipt" + assert_grep 'x_request=req-remote-nonwritable' "$meta" \ + "a non-writable remote state must leave the link untouched" + pass "retire fails closed when remote state is non-writable" +} + +test_remote_retire_accepts_nonwritable_absence() { + local home remote rc + remote_fixture_prepare + home=$(make_home remote-no-link) + remote=$(make_remote_route "$home" mate) + seed_repro_commitment "$home" pf-remote-no-link req-remote-no-link secondmate:mate work-no-link + fm_write_meta "$remote/state/work-no-link.meta" "status=done" + chmod 555 "$remote/state" + + rc=0 + run_pf_remote "$home" retire pf-remote-no-link --reason "already cleared" --force >/dev/null 2>&1 || rc=$? + chmod 700 "$remote/state" + [ "$rc" -eq 0 ] || fail "retire must accept an absent link without requiring write access" + assert_present "$home/state/public-followup/retired/pf-remote-no-link" \ + "an absent link must permit a retirement receipt" + assert_absent "$home/state/public-followup/registry/pf-remote-no-link" \ + "an absent link must close the registration" + assert_no_grep 'x_request=' "$remote/state/work-no-link.meta" \ + "an already-cleared remote task must remain unlinked" + pass "retire accepts link absence in non-writable remote state" +} + +# The guarded clear runs unattended over the transport, so it must REFUSE rather +# than wedge when it cannot take the metadata lock. The writability precondition +# narrows that window but cannot close it: the parent can turn non-writable +# between that check and lock creation, and a lock held by a live holder is +# indistinguishable from it at the acquire. The ordinary wait retries forever, so +# before the bounded acquire this path hung instead of returning the +# reconciliation refusal, leaving deliver or retire stuck with nothing reported. +# +# The state directory is deliberately left WRITABLE here, so a refusal can only +# come from the bounded lock wait and never from the writability precondition. +test_remote_retire_refuses_unacquirable_lock_without_hanging() { + local home remote meta lock holder rc started elapsed + remote_fixture_prepare + home=$(make_home remote-lock-bound) + remote=$(make_remote_route "$home" mate) + seed_repro_commitment "$home" pf-remote-lock req-remote-lock secondmate:mate work-lock + meta="$remote/state/work-lock.meta" + fm_write_meta "$meta" \ + "status=working" "x_request=req-remote-lock" "x_request_ts=1700000000" "x_followups=1" + + # A lock held by a genuinely live process: it cannot be reclaimed as stale, so + # the acquire can never succeed and only a bound can end the wait. + sleep 300 & + holder=$! + PF_TEST_LOCK_HOLDER=$holder + lock="$remote/state/.meta-work-lock.lock" + mkdir -p "$lock" + printf '%s\n' "$holder" > "$lock/pid" + + started=$(date +%s) + rc=0 + EXPECT_OUT=$(run_pf_remote_timed 30 "$home" retire pf-remote-lock --reason "lock held" --force 2>&1) || rc=$? + elapsed=$(( $(date +%s) - started )) + kill "$holder" 2>/dev/null || true + wait "$holder" 2>/dev/null || true + PF_TEST_LOCK_HOLDER= + rm -rf "$lock" + + [ "$rc" -ne 124 ] \ + || fail "the guarded clear exceeded the test harness deadline" + [ "$rc" -ne 0 ] || fail "retire must refuse when the metadata lock cannot be acquired" + # The bound is what this case exists to prove. An unbounded wait reaches the + # independent harness deadline instead of this observable refusal. + [ "$elapsed" -lt 30 ] \ + || fail "the guarded clear did not return promptly; it waited ${elapsed}s for an unacquirable lock" + assert_contains "$EXPECT_OUT" "could not clear the legacy X link" \ + "an unacquirable lock must use the retained reconciliation refusal" + assert_present "$home/state/public-followup/registry/pf-remote-lock" \ + "an unacquirable lock must retain the registration" + assert_absent "$home/state/public-followup/retired/pf-remote-lock" \ + "an unacquirable lock must not write a retirement receipt" + assert_grep 'x_request=req-remote-lock' "$meta" \ + "an unacquirable lock must leave the remote link untouched" + pass "the guarded remote clear refuses a lock it cannot acquire instead of hanging" +} + +# fm-on.sh passes ssh's status through, so 255 is unknown remote completion, not +# proof the clear failed. The close must refuse and retain rather than either +# claiming the link is gone or reporting a definite failure, so reconciliation +# lands on the host that actually owns the answer. +test_remote_unconfirmed_clear_is_unknown_completion() { + local home remote + remote_fixture_prepare + home=$(make_home remote-unknown) + remote=$(make_remote_route "$home" mini-default) + seed_repro_commitment "$home" pf-remote-unknown req-remote-unknown secondmate:mini-default work-unknown + fm_write_meta "$remote/state/work-unknown.meta" \ + "x_request=req-remote-unknown" "x_request_ts=1700000000" "x_followups=1" + + FM_FAKE_SSH_MODE=unreachable \ + expect_failure "an unconfirmed remote clear must not close the loop" \ + run_pf_remote "$home" retire pf-remote-unknown --reason "closing" --force + assert_contains "$EXPECT_OUT" "could not clear the legacy X link" \ + "an unconfirmed remote clear must still report the link as unresolved" + assert_contains "$EXPECT_OUT" "never confirmed the clear" \ + "unknown remote completion must be named, not reported as a definite failure" + assert_present "$home/state/public-followup/registry/pf-remote-unknown" \ + "unknown remote completion must retain the registration" + assert_absent "$home/state/public-followup/retired/pf-remote-unknown" \ + "unknown remote completion must not record a retirement receipt" + assert_grep 'x_request=req-remote-unknown' "$remote/state/work-unknown.meta" \ + "an unreachable host must leave the remote link exactly as it was" + pass "an unconfirmed remote clear is unknown completion, never a silent close" +} + # CI's stock macOS Bash lane sets FM_TEST_ONLY to run just the bash-3.2 empty-lock # register regression. The rest of this file is not a 3.2 snapshot suite. if [ -n "${FM_TEST_ONLY:-}" ]; then @@ -2332,3 +2744,11 @@ test_brief_fails_without_typed_deliverable_keys test_prechange_registration_is_open_and_unrechainable test_x_request_teardown_warns_when_final_unposted test_secondmate_promotion_uses_teardown_parent_resolution +test_remote_secondmate_loop_delivers_and_retires +test_remote_retire_force_semantics_unchanged +test_remote_retire_refuses_reassigned_route +test_remote_retire_refuses_unreadable_state +test_remote_retire_refuses_nonwritable_state +test_remote_retire_accepts_nonwritable_absence +test_remote_retire_refuses_unacquirable_lock_without_hanging +test_remote_unconfirmed_clear_is_unknown_completion