fix(remote): keep fm-remote Herdr servers in the Aqua session

bin/fm-remote-doctor.sh

@@ -13,9 +13,15 @@
 # fm-remote session. Its account therefore needs the Firstmate-owned Aqua Herdr
 # agent plus the sibling dev.firstmate.remote-job worker that runs normal fm-on
 # commands through the Aqua or Linux job-worker path. On darwin, that Herdr
-# agent starts the server through the remote account's login shell (`-l -c`)
-# so the Aqua login session gets login-shell environment and login-keychain
-# access; exec keeps herdr in the foreground under launchd. Doctor remains
+# agent runs bin/fm-remote-herdr-guard.sh through the remote account's login
+# shell (`-l -c`) so the server inherits the account's own environment; the
+# gui/<uid> launchd domain it is bootstrapped into, not the shell, is what
+# gives the server and its panes the Aqua audit session and login-keychain
+# access. The guard execs the server in the foreground under launchd, leaves an
+# Aqua-born server alone, and takes the session over from a server born
+# outside that session (an SSH remote attach wins the socket at boot), because
+# such a server's panes cannot read the login keychain;
+# bin/fm-remote-herdr-owner-lib.sh owns that birth test. Doctor remains
 # invokable over the plain-SSH bootstrap path to inspect and repair that worker.
 # SSH cannot create an Aqua session, so a host with no GUI login is a human
 # gap rather than something --fix attempts to bypass.
@@ -59,6 +65,8 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(CDPATH='' cd "$SCRIPT_DIR/.." && pwd -P)}"
 . "$SCRIPT_DIR/fm-remote-job-lib.sh"
 # shellcheck source=bin/fm-tasks-axi-lib.sh
 . "$SCRIPT_DIR/fm-tasks-axi-lib.sh"
+# shellcheck source=bin/fm-remote-herdr-owner-lib.sh
+. "$SCRIPT_DIR/fm-remote-herdr-owner-lib.sh"
 REQUIRED_TOOLS=(git jq herdr tasks-axi treehouse)
 HARNESS_TOOLS=(claude codex opencode pi pi-signed grok kimi)
 OPTIONAL_TOOLS=(tmux no-mistakes gh)
@@ -152,14 +160,47 @@ herdr_adapter_load() {
   FM_REMOTE_DOCTOR_HERDR_LOADED=1
 }
 
+herdr_server_status_json() {
+  herdr_adapter_load || return 1
+  fm_backend_herdr_cli "$HERDR_SESSION_NAME" status --json 2>/dev/null
+}
+
 herdr_server_running() {
   local running
-  herdr_adapter_load || return 1
-  running=$(fm_backend_herdr_cli "$HERDR_SESSION_NAME" status --json 2>/dev/null \
-    | jq -r '.server.running // false' 2>/dev/null) || return 1
+  running=$(herdr_server_status_json | jq -r '.server.running // false' 2>/dev/null) || return 1
   [ "$running" = true ]
 }
 
+# Birth of the process serving the session, as the guard classifies it:
+# prints "<birth> <pid>" (launchd, worker, ssh, or unknown), "unproven" when
+# no herdr process can be shown to hold the socket, or "nolsof" when lsof does
+# not resolve. bin/fm-remote-herdr-owner-lib.sh owns the markers.
+herdr_server_birth() {
+  local socket owner rc birth
+  socket=$(herdr_server_status_json | jq -r '.server.socket // empty' 2>/dev/null) || socket=
+  owner=$(fm_remote_herdr_socket_owner "$socket"); rc=$?
+  if [ "$rc" -eq 2 ]; then
+    printf 'nolsof\n'
+    return 0
+  fi
+  if [ -z "$owner" ]; then
+    printf 'unproven\n'
+    return 0
+  fi
+  birth=$(fm_remote_herdr_owner_birth "$owner")
+  printf '%s %s\n' "$birth" "$owner"
+}
+
+# On darwin the session is ready only when its server was born in the Aqua
+# login session; elsewhere any running server is.
+herdr_server_aqua_owned() {
+  local birth
+  herdr_server_running || return 1
+  [ "$PLATFORM" = darwin ] || return 0
+  birth=$(herdr_server_birth)
+  fm_remote_herdr_birth_is_aqua "${birth%% *}"
+}
+
 launch_agent_is_aqua() {
   local stripped
   [ -f "$LAUNCH_AGENT_PLIST" ] && [ ! -L "$LAUNCH_AGENT_PLIST" ] || return 1
@@ -209,14 +250,20 @@ resolve_launch_agent_shell() {
   printf '%s' /bin/sh
 }
 
-# Login-shell command that execs the resolved herdr so launchd keeps one
-# foreground process in the Aqua session (login-keychain access) instead of
-# letting herdr self-daemonize into a Background session. KeepAlive stays
-# unconditionally true: this agent uniquely owns fm-remote, and a
-# SuccessfulExit=false plus busy-socket no-op would change the existing
-# restart-on-any-exit contract.
+# Login-shell command that execs the Firstmate-owned guard, which in turn execs
+# the resolved herdr so launchd keeps one foreground process in the Aqua
+# session, or exits 0 when an Aqua-born server already owns the session.
+# KeepAlive={SuccessfulExit=false} is load-bearing for that exit: an
+# unconditional KeepAlive would respawn the job every throttle interval
+# forever while a foreign server holds the socket, exactly the loop this guard
+# replaces, and would never let the guard's "nothing to do" verdict rest.
+launch_agent_guard_path() {
+  printf '%s/bin/fm-remote-herdr-guard.sh' "$FM_ROOT"
+}
+
 launch_agent_exec_command() { # <resolved-herdr-path>
-  printf 'exec %s server --session %s' \
+  printf 'exec %s %s %s' \
+    "$(launch_agent_shell_quote "$(launch_agent_guard_path)")" \
     "$(launch_agent_shell_quote "$1")" \
     "$(launch_agent_shell_quote "$HERDR_SESSION_NAME")"
 }
@@ -244,7 +291,12 @@ render_launch_agent() { # <resolved-herdr-path> <resolved-login-shell>
 	<key>RunAtLoad</key>
 	<true/>
 	<key>KeepAlive</key>
-	<true/>
+	<dict>
+		<key>SuccessfulExit</key>
+		<false/>
+	</dict>
+	<key>ThrottleInterval</key>
+	<integer>10</integer>
 	<key>StandardOutPath</key>
 	<string>$LAUNCH_AGENT_LOG</string>
 	<key>StandardErrorPath</key>
@@ -278,7 +330,10 @@ launch_agent_loaded_contract_matches() { # <resolved-login-shell>
   [[ "$loaded" == *"$args"* ]] || return 1
   [[ "$loaded" == *"stdoutpath=$log_compact"* ]] || return 1
   [[ "$loaded" == *"stderrpath=$log_compact"* ]] || return 1
-  [[ "$loaded" == *'properties=keepalive|runatload'* ]] || return 1
+  # launchd renders KeepAlive={SuccessfulExit=false} as a successful-exit
+  # semaphore rather than a keepalive property.
+  [[ "$loaded" == *'successfulexit=>0'* ]] || return 1
+  [[ "$loaded" == *'properties=runatload'* ]] || return 1
 }
 
 # --- remote job and tool checks ---------------------------------------------
@@ -609,7 +664,29 @@ check_herdr_server() {
     return 0
   fi
   if herdr_server_running; then
-    record herdr-server "ok: session $HERDR_SESSION_NAME is running"
+    if [ "$PLATFORM" != darwin ]; then
+      record herdr-server "ok: session $HERDR_SESSION_NAME is running"
+      return 0
+    fi
+    local birth
+    birth=$(herdr_server_birth)
+    case "$birth" in
+      launchd\ *|worker\ *)
+        record herdr-server "ok: session $HERDR_SESSION_NAME is running in the Aqua login session (pid ${birth#* }, ${birth%% *})"
+        ;;
+      nolsof)
+        record herdr-server "human: session $HERDR_SESSION_NAME is running but lsof does not resolve, so its server's birth cannot be proven" \
+          "install lsof on that account so the launch agent and this check can tell an Aqua-born server from one started over SSH"
+        ;;
+      unproven)
+        record herdr-server "fixable: session $HERDR_SESSION_NAME is running but no herdr process can be shown to own its socket, so its birth cannot be proven" \
+          "rerun this command with --fix so the launch agent takes the session over (its current panes close and the parent firstmate relaunches its mates)"
+        ;;
+      *)
+        record herdr-server "fixable: session $HERDR_SESSION_NAME is served by pid ${birth#* } born outside the Aqua login session (${birth%% *}), so its panes cannot reach the login keychain" \
+          "rerun this command with --fix so the launch agent takes the session over (its current panes close and the parent firstmate relaunches its mates)"
+        ;;
+    esac
     return 0
   fi
   if [ "$PLATFORM" = darwin ] && ! check_is_ok gui-session; then
@@ -685,7 +762,7 @@ write_launch_agent() { # <resolved-login-shell>
     fix_report launchagent failed "cannot publish $LAUNCH_AGENT_PLIST"
     return 1
   fi
-  fix_report launchagent applied "wrote the Aqua-scoped $LAUNCH_AGENT_LABEL launch agent running $herdr_bin server via $shell -l -c"
+  fix_report launchagent applied "wrote the Aqua-scoped $LAUNCH_AGENT_LABEL launch agent running $(launch_agent_guard_path) for $herdr_bin via $shell -l -c"
 }
 
 # Reload rather than plain bootstrap so a rewritten plist replaces a stale
@@ -711,7 +788,7 @@ reload_launch_agent() { # <check-to-report-under>
     return 1
   fi
   if ! wait_for_herdr_server; then
-    fix_report "$report" failed "the herdr server for session $HERDR_SESSION_NAME did not report running within 10s"
+    fix_report "$report" failed "the herdr server for session $HERDR_SESSION_NAME did not come up inside the Aqua launch agent within 10s"
     return 1
   fi
   fix_report "$report" applied "bootstrapped and started $LAUNCH_AGENT_LABEL in gui/$UID_NUM"
@@ -720,7 +797,7 @@ reload_launch_agent() { # <check-to-report-under>
 wait_for_herdr_server() {
   local i=0
   while [ "$i" -lt 20 ]; do
-    herdr_server_running && return 0
+    herdr_server_aqua_owned && return 0
     i=$((i + 1))
     sleep 0.5
   done

bin/fm-remote-entrypoint.sh

@@ -31,7 +31,7 @@
 set -eu
 
 PROTOCOL=1
-DOCTOR_SHA256=810316fbb621f2ca9cdad0d085dee44a09b70f403b2056c9c6971ac7e767104a
+DOCTOR_SHA256=78efccd6cb7a0123400e49fa323292a64c8e3c7ebd3717151be69f87735302fb
 REAL_SOURCE=$(python3 -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' "${BASH_SOURCE[0]}" 2>/dev/null) ||
   REAL_SOURCE=$(realpath "${BASH_SOURCE[0]}" 2>/dev/null) ||
   REAL_SOURCE=${BASH_SOURCE[0]}

bin/fm-remote-herdr-guard.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# launchd exec target for the Firstmate-owned dev.firstmate.herdr.fm-remote
+# launch agent: make the Aqua login session own the fm-remote Herdr server.
+#
+# Usage:
+#   fm-remote-herdr-guard.sh <herdr-path> <session>
+#
+# bin/fm-remote-doctor.sh renders the launch agent as the account's login
+# shell running `exec <this script> <herdr> fm-remote` with
+# LimitLoadToSessionType=Aqua, RunAtLoad, KeepAlive={SuccessfulExit=false},
+# and ThrottleInterval=10, then bootstraps it into gui/<uid>. That domain, not
+# the login shell, is what gives this process and every server it execs the
+# Aqua audit session and login-keychain access; the login shell only gives the
+# server the account's own environment.
+# `herdr server` stays in the foreground under launchd, as verified in
+# docs/verification/runtime-backends.md under "fm-remote server birth and login-keychain access", so the final exec provides the complete supervision lifecycle.
+#
+# Decision, made once per launch (exit codes matter under SuccessfulExit=false:
+# 0 tells launchd the job is done until something restarts it, non-zero asks
+# for a retry after the throttle interval):
+#   no server owns the session socket  -> exec `herdr server --session <s>`
+#                                          (foreground, launchd-supervised)
+#   the owner was born in the Aqua session (launchd or the Aqua remote-job
+#   worker)                            -> exit 0, leave it alone
+#   the owner was born anywhere else (an SSH remote attach, a shell over
+#   ssh/mosh, or a birth it cannot prove) -> `herdr server stop`, wait until the
+#                                          socket is released, then exec
+#                                          `herdr server --session <s>` at once
+#                                          so the socket is rebound before a
+#                                          reconnecting SSH attach can start
+#                                          another foreign server
+#   the foreign server does not release the socket in time -> exit 1
+# A takeover closes every pane in that session; the parent firstmate's
+# secondmate liveness sweep relaunches its mates into the Aqua-born server.
+# bin/fm-remote-herdr-owner-lib.sh owns the owner discovery and the birth
+# markers; FM_REMOTE_HERDR_GUARD_STOP_WAIT_TENTHS (default 50) bounds the
+# release wait in tenths of a second. Every decision prints one line to
+# stdout, which launchd routes to the agent's log.
+set -u
+
+SCRIPT_SELF=${BASH_SOURCE[0]}
+SCRIPT_DIR=${SCRIPT_SELF%/*}
+[ "$SCRIPT_DIR" != "$SCRIPT_SELF" ] || SCRIPT_DIR=.
+SCRIPT_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR" && pwd -P)
+# shellcheck source=bin/fm-remote-herdr-owner-lib.sh
+. "$SCRIPT_DIR/fm-remote-herdr-owner-lib.sh"
+
+usage() { sed -n '2,6p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
+[ "$#" -eq 2 ] || usage
+HERDR_BIN=$1
+SESSION=$2
+[ -n "$HERDR_BIN" ] && [ -x "$HERDR_BIN" ] || { printf 'fm-remote-herdr-guard: herdr is not executable: %s\n' "$HERDR_BIN" >&2; exit 1; }
+[ -n "$SESSION" ] || usage
+command -v jq >/dev/null 2>&1 || { printf 'fm-remote-herdr-guard: jq does not resolve on the launch agent PATH\n' >&2; exit 1; }
+STOP_WAIT_TENTHS=${FM_REMOTE_HERDR_GUARD_STOP_WAIT_TENTHS:-50}
+
+log() { printf 'fm-remote-herdr-guard: %s\n' "$*"; }
+
+herdr_status() { # prints the session's status JSON, empty when herdr fails
+  HERDR_SESSION="$SESSION" "$HERDR_BIN" status --json --session "$SESSION" 2>/dev/null || true
+}
+
+status_running() { # <status-json>
+  [ "$(printf '%s' "$1" | jq -r '.server.running // false' 2>/dev/null)" = true ]
+}
+
+start_server() {
+  log "starting the herdr server for session $SESSION inside this launch agent (pid $$)"
+  exec "$HERDR_BIN" server --session "$SESSION"
+}
+
+STATUS=$(herdr_status)
+if ! status_running "$STATUS"; then
+  log "no server owns session $SESSION"
+  start_server
+fi
+
+SOCKET=$(printf '%s' "$STATUS" | jq -r '.server.socket // empty' 2>/dev/null)
+OWNER=$(fm_remote_herdr_socket_owner "$SOCKET"); OWNER_RC=$?
+if [ "$OWNER_RC" -eq 2 ]; then
+  log "session $SESSION is running but lsof does not resolve, so its server's birth cannot be proven"
+  BIRTH=unknown
+elif [ -z "$OWNER" ]; then
+  log "session $SESSION is running but no herdr process could be proven to own ${SOCKET:-its socket}"
+  BIRTH=unknown
+else
+  BIRTH=$(fm_remote_herdr_owner_birth "$OWNER")
+fi
+
+if fm_remote_herdr_birth_is_aqua "$BIRTH"; then
+  log "session $SESSION is served by pid $OWNER born in the Aqua login session ($BIRTH); nothing to do"
+  exit 0
+fi
+
+log "session $SESSION is served by ${OWNER:+pid }${OWNER:-an unproven process} born outside the Aqua login session ($BIRTH); its panes cannot reach the login keychain, taking the session over"
+HERDR_SESSION="$SESSION" "$HERDR_BIN" server stop --session "$SESSION" >/dev/null 2>&1 \
+  || log "herdr server stop for session $SESSION did not succeed; waiting for the socket anyway"
+i=0
+while [ "$i" -lt "$STOP_WAIT_TENTHS" ]; do
+  if ! status_running "$(herdr_status)"; then
+    log "session $SESSION released its socket after $i tenths of a second"
+    start_server
+  fi
+  sleep 0.1
+  i=$((i + 1))
+done
+log "the foreign server for session $SESSION did not release its socket within $STOP_WAIT_TENTHS tenths of a second; exiting 1 so launchd retries"
+exit 1