From 733a5047f947d7df10ad9cea4f7a501bc706de3a Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Mon, 3 Aug 2026 17:10:59 -0700 Subject: [PATCH 1/8] feat(bin): preflight remote runtime tool paths (#1623) * feat(bin): widen the remote runtime PATH and add a remote doctor preflight The fixed remote entrypoint hard-coded a four-directory PATH, so a remote account whose tools live under nix or a per-user profile could not run basic Firstmate work without a login shell. The entrypoint now composes its child PATH from the code root's bin, the account's ~/.local/bin, the common package-manager directories that actually exist on the host, and the portable system tail, deduplicated and in a fixed order, still under env -i with the same variable allowlist and no shell command string. fm-remote-doctor.sh reports that exact PATH by inheriting it from its own entrypoint launch rather than recomposing it, so the ordering keeps one owner. It is read-only, reports where each required and optional tool resolved, and exits non-zero naming every required tool that did not. Remote seeding runs it as a preflight before anything is created on the host and restores the registry when it fails. * no-mistakes(review): Harden remote git authorization and missing-tool diagnostics * no-mistakes(document): Document remote PATH doctor and safe shims * no-mistakes(lint): Fix ShellCheck findings in remote path tests * no-mistakes(lint): Suppress exported fixture's false-positive ShellCheck warning --- bin/fm-remote-doctor.sh | 53 ++++++++ bin/fm-remote-entrypoint.sh | 72 +++++++++- bin/fm-remote-home-seed.sh | 26 +++- docs/remote-secondmates.md | 45 +++++- docs/scripts.md | 1 + tests/fm-on.test.sh | 128 +++++++++++++++++- ...fm-remote-secondmate-lifecycle-e2e.test.sh | 24 ++++ 7 files changed, 338 insertions(+), 11 deletions(-) create mode 100755 bin/fm-remote-doctor.sh diff --git a/bin/fm-remote-doctor.sh b/bin/fm-remote-doctor.sh new file mode 100755 index 00000000000..fad8d09d0d7 --- /dev/null +++ b/bin/fm-remote-doctor.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Report the runtime PATH and tool resolution of a Firstmate remote account. +# +# Usage: +# bin/fm-on.sh fm-remote-doctor.sh +# +# Read-only preflight. Run it through fm-on.sh so the fixed remote entrypoint +# composes and exports the child PATH: this command reports the PATH it inherits +# instead of recomposing it, so the entrypoint stays the single owner of that +# ordering and the two can never drift. Required tools that do not resolve are +# listed and set a non-zero exit status; optional tools are reported either way +# and never fail the check. Nothing on the host is created or modified. +set -eu + +REQUIRED_TOOLS=(git) +OPTIONAL_TOOLS=(tmux treehouse no-mistakes tasks-axi herdr claude codex opencode pi grok kimi) + +usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } + +[ "$#" -eq 0 ] || usage + +printf 'path=%s\n' "${PATH:-}" +if [ -n "${FM_ROOT_OVERRIDE:-}" ] && [ "${PATH%%:*}" = "$FM_ROOT_OVERRIDE/bin" ]; then + printf 'entrypoint=yes\n' +else + printf 'entrypoint=no\n' + printf 'note: not launched through the fixed remote entrypoint; the reported PATH is this caller environment.\n' >&2 +fi + +MISSING=() +for tool in "${REQUIRED_TOOLS[@]}"; do + if resolved=$(command -v "$tool" 2>/dev/null); then + printf 'required %s=%s\n' "$tool" "$resolved" + else + printf 'required %s=MISSING\n' "$tool" + MISSING+=("$tool") + fi +done +for tool in "${OPTIONAL_TOOLS[@]}"; do + if resolved=$(command -v "$tool" 2>/dev/null); then + printf 'optional %s=%s\n' "$tool" "$resolved" + else + printf 'optional %s=absent\n' "$tool" + fi +done + +if [ "${#MISSING[@]}" -gt 0 ]; then + printf 'error: required tools do not resolve on the remote runtime PATH: %s\n' "${MISSING[*]}" >&2 + printf 'fix: install each one where it resolves on the path reported above, or put a wrapper script for it in %s/.local/bin, which is always on that PATH.\n' "${HOME:-~}" >&2 + printf 'fix: tools provided by nvm, asdf, or mise never resolve here because no login or interactive shell runs; see docs/remote-secondmates.md for the wrapper recipe.\n' >&2 + exit 1 +fi +printf 'ok: every required tool resolves on the remote runtime PATH\n' diff --git a/bin/fm-remote-entrypoint.sh b/bin/fm-remote-entrypoint.sh index 5e2dfea3d63..f977511de92 100755 --- a/bin/fm-remote-entrypoint.sh +++ b/bin/fm-remote-entrypoint.sh @@ -11,13 +11,64 @@ # for protocol framing, so it remains byte-for-byte available to the command. # The child receives an empty environment plus fixed PATH, HOME, FM_HOME, and # FM_ROOT_OVERRIDE. stdout, stderr, and exit status pass through unchanged. +# +# This file is the single owner of the child PATH. compose_operator_path builds +# its operator portion from the account's ~/.local/bin, the common +# package-manager directories that exist on this host, and the always-present +# system tail. The tracked-command check resolves git from that portion before +# /bin is prepended for the child. No login or interactive shell is ever +# started, so a tool that +# lives outside those directories - anything installed by nvm, asdf, or mise - +# needs a wrapper in ~/.local/bin. bin/fm-remote-doctor.sh reports this exact +# PATH by inheriting it rather than recomposing it, so the two cannot drift. set -eu PROTOCOL=1 -SAFE_PATH='/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin' +OPERATOR_PATH= die() { printf 'error: %s\n' "$1" >&2; exit "${2:-64}"; } +path_append() { # + case ":$OPERATOR_PATH:" in *":$1:"*) return 0 ;; esac + OPERATOR_PATH="${OPERATOR_PATH:+$OPERATOR_PATH:}$1" +} + +path_append_if_dir() { # + [ -d "$1" ] || return 0 + path_append "$1" +} + +build_child_path() { # + local root_bin=$1 directory old_ifs + CHILD_PATH=$root_bin + old_ifs=$IFS + IFS=: + for directory in $OPERATOR_PATH; do + case ":$CHILD_PATH:" in *":$directory:"*) continue ;; esac + CHILD_PATH="$CHILD_PATH:$directory" + done + IFS=$old_ifs +} + +compose_operator_path() { # + local account_home=$1 account_user + OPERATOR_PATH= + path_append "$account_home/.local/bin" + path_append_if_dir "$account_home/.nix-profile/bin" + # env -i clears USER, so ask the password database rather than the environment. + account_user=$(id -un 2>/dev/null) || account_user= + if [ -n "$account_user" ]; then + path_append_if_dir "/etc/profiles/per-user/$account_user/bin" + fi + path_append_if_dir /run/current-system/sw/bin + path_append_if_dir /opt/homebrew/bin + path_append_if_dir /usr/local/bin + path_append /usr/bin + path_append /bin + path_append /usr/sbin + path_append /sbin +} + base64_decode_to() { # local encoded=$1 destination=$2 if printf '%s' "$encoded" | base64 --decode > "$destination" 2>/dev/null; then @@ -119,15 +170,26 @@ case "$COMMAND" in */*|*..*) die "command contains a path or traversal: $COMMAND COMMAND_PATH="$ROOT/bin/$COMMAND" [ -f "$COMMAND_PATH" ] && [ ! -L "$COMMAND_PATH" ] && [ -x "$COMMAND_PATH" ] \ || die "command is not a genuine executable in the configured remote root: $COMMAND" -git -C "$ROOT" ls-files --error-unmatch "bin/$COMMAND" >/dev/null 2>&1 \ - || die "command is not tracked by the configured remote root: $COMMAND" - unset HOME ACCOUNT_HOME=$(CDPATH='' cd ~ 2>/dev/null && pwd -P) || die "cannot resolve the remote account home" +compose_operator_path "$ACCOUNT_HOME" +GIT_BIN=$(PATH="$OPERATOR_PATH" command -v git 2>/dev/null || true) +case "$GIT_BIN" in + /*) + [ -x "$GIT_BIN" ] || GIT_BIN= + case ":$OPERATOR_PATH:" in *":${GIT_BIN%/*}:"*) ;; *) GIT_BIN= ;; esac + ;; + *) GIT_BIN= ;; +esac +[ -n "$GIT_BIN" ] || die "required tool git does not resolve on the remote operator PATH; install git there or put a wrapper for it in ~/.local/bin using the recipe in docs/remote-secondmates.md" +"$GIT_BIN" -C "$ROOT" ls-files --error-unmatch "bin/$COMMAND" >/dev/null 2>&1 \ + || die "command is not tracked by the configured remote root: $COMMAND" +build_child_path "$ROOT/bin" + trap - EXIT rm -rf -- "$TMP" exec /usr/bin/env -i \ - PATH="$ROOT/bin:$ACCOUNT_HOME/.local/bin:$SAFE_PATH" \ + PATH="$CHILD_PATH" \ HOME="$ACCOUNT_HOME" \ FM_HOME="$HOME_PATH" \ FM_ROOT_OVERRIDE="$ROOT" \ diff --git a/bin/fm-remote-home-seed.sh b/bin/fm-remote-home-seed.sh index 849b32a05b2..e0e723434ac 100755 --- a/bin/fm-remote-home-seed.sh +++ b/bin/fm-remote-home-seed.sh @@ -6,7 +6,8 @@ # # The SSH alias must already reach a host whose non-interactive PATH exposes the # fixed fm-remote-entrypoint.sh from . The command records the -# remote host dimension in data/secondmates.md, sends a bounded provisioning +# remote host dimension in data/secondmates.md, preflights the remote runtime +# with fm-remote-doctor.sh before touching that host, sends a bounded provisioning # manifest through fm-on.sh, and lets the remote host clone its own Firstmate # home and project origins. No project tree or secret environment is copied. # Known provisioning failure rolls the registry back. SSH status 255 preserves @@ -30,7 +31,7 @@ MAX_MANIFEST_BYTES=1048576 . "$SCRIPT_DIR/fm-wake-lib.sh" die() { printf 'error: %s\n' "$1" >&2; exit 1; } -usage() { sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } +usage() { sed -n '2,14p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } encode() { base64 | tr -d '\n'; } safe_id() { case "$1" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac; } @@ -178,14 +179,31 @@ if ! secondmate_registry_validate_bindings "$REG" secondmate_registry_path_key " die "$SECONDMATE_REGISTRY_ERROR" fi +restore_registry_and_brief() { + if [ "$REG_EXISTED" -eq 1 ]; then cp "$TMP/registry.before" "$REG"; else rm -f -- "$REG"; fi + [ "$BRIEF_CREATED" -eq 0 ] || rm -f -- "$BRIEF" +} + +# Preflight the remote runtime before anything is created on that host. The +# doctor runs through the same fixed entrypoint as every later call, so it sees +# the exact PATH the remote home will run under. +set +e +PREFLIGHT_OUT=$("$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-doctor.sh 2>&1) +PREFLIGHT_RC=$? +set -e +if [ "$PREFLIGHT_RC" -ne 0 ]; then + restore_registry_and_brief + [ -z "$PREFLIGHT_OUT" ] || printf '%s\n' "$PREFLIGHT_OUT" >&2 + die "remote runtime preflight failed; nothing was provisioned. Fix the reported tools, or update the remote code root if it predates fm-remote-doctor.sh" +fi + set +e PROVISION_OUT=$("$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-home-provision.sh < "$TMP/manifest" 2>&1) PROVISION_RC=$? set -e if [ "$PROVISION_RC" -ne 0 ]; then if [ "$PROVISION_RC" -ne 255 ]; then - if [ "$REG_EXISTED" -eq 1 ]; then cp "$TMP/registry.before" "$REG"; else rm -f -- "$REG"; fi - [ "$BRIEF_CREATED" -eq 0 ] || rm -f -- "$BRIEF" + restore_registry_and_brief fi [ -z "$PROVISION_OUT" ] || printf '%s\n' "$PROVISION_OUT" >&2 if [ "$PROVISION_RC" -eq 255 ]; then diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index 1f606b3cff7..bb87d82f94c 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -25,6 +25,48 @@ It never accepts a shell command string and starts the selected script with a mi The remote account must provide Firstmate's universal toolchain, the selected worker runtime, the selected session backend, and credentials that work on that host. Project origin URLs recorded by the primary must be reachable from the remote account because projects are cloned on that host rather than copied from the primary. +## Non-interactive tool contract + +No login or interactive shell ever runs on the remote host, so `~/.profile`, `~/.bashrc`, and `~/.zshrc` never contribute to the runtime `PATH`. +`bin/fm-remote-entrypoint.sh` is the single owner of the `PATH` its children receive and composes it in this order: + +1. `/bin`. +2. The account's `~/.local/bin`, always included so an account can add tools without changing this contract. +3. Each of `~/.nix-profile/bin`, `/etc/profiles/per-user//bin`, `/run/current-system/sw/bin`, `/opt/homebrew/bin`, and `/usr/local/bin`, in that order, and only when the directory exists on the host. +4. The system tail `/usr/bin:/bin:/usr/sbin:/sbin`. + +Repeated directories are collapsed to their first position, so the order above is exactly what a remote command sees. +The entrypoint resolves `git` from the operator directories in steps 2 through 4 for tracked-command authorization, then prepends `/bin` only for the authorized child. +A checkout-local `bin/git` therefore cannot authorize an untracked command, and a host with no operator `git` receives an install-or-wrapper diagnostic before command execution. + +A tool that only exists inside a version manager - nvm, asdf, or mise - never resolves under that contract, because those managers publish their shims through shell initialization. +The supported escape hatch is a wrapper in `~/.local/bin` rather than special-casing a version manager inside the entrypoint: + +```sh +mkdir -p ~/.local/bin +cat > ~/.local/bin/tasks-axi <<'SH' +#!/usr/bin/env bash +tool_bin="$HOME/.nvm/versions/node//bin" +PATH="$tool_bin:$PATH" +exec "$tool_bin/tasks-axi" "$@" +SH +chmod +x ~/.local/bin/tasks-axi +``` + +Replace the placeholder with the remote account's selected nvm version. +For asdf or mise, use the same shape with the selected version's absolute `bin` directory, one wrapper per tool the remote home actually needs. +The wrapper must execute that absolute target rather than resolving its own name again through `~/.local/bin`. + +Check any host against the real contract: + +```sh +bin/fm-on.sh fm-remote-doctor.sh +``` + +The doctor is read-only. +It prints the exact `PATH` its own entrypoint launch produced, then reports where each required and optional tool resolved. +It exits non-zero and names every required tool that did not resolve, so the output is the install-or-shim list for that host. + ## Provision a route Create and fill the normal secondmate charter first, then run: @@ -35,7 +77,8 @@ bin/fm-remote-home-seed.sh {` is the remote Firstmate code clone that supplies tracked scripts. `` is a separate absolute path for the persistent secondmate home and must not overlap the code root. -The seed records `host:`, `root:`, and `home:` in `data/secondmates.md`, sends a bounded manifest, and lets the remote host clone its own Firstmate home and project origins. +The seed records `host:`, `root:`, and `home:` in `data/secondmates.md`, preflights the host with `fm-remote-doctor.sh`, sends a bounded manifest, and lets the remote host clone its own Firstmate home and project origins. +A failing preflight prints the doctor's missing-tool list, restores the registry, and creates nothing on the remote host. It does not copy project trees or the primary process environment. A known provisioning failure rolls back the new route, while SSH exit 255 preserves it because remote completion is unknown and must be reconciled on the same host. diff --git a/docs/scripts.md b/docs/scripts.md index ac64b904e49..82e1e585d32 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -17,6 +17,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-bearings-snapshot.sh` | Project the fleet snapshot to the compact TOON bearings view; local-only unless `--include-prs` | | `fm-update.sh` | Fast-forward-only self-update of firstmate and local or remote secondmate homes | | `fm-on.sh` | Execute one tracked Firstmate command in a configured remote secondmate home | +| `fm-remote-doctor.sh` | Report the inherited remote runtime PATH and required or optional tool resolution | | `fm-backlog-handoff.sh` | Validate and delegate queued backlog-item moves into a secondmate home | | `fm-backlog-receive.sh` | Idempotently ingest one confined remote handoff outbox through tasks-axi | | `fm-decision-hold.sh` | Create, verify, complete, and resolve durable captain-held decisions | diff --git a/tests/fm-on.test.sh b/tests/fm-on.test.sh index 4977f3a383a..3d041b2a681 100755 --- a/tests/fm-on.test.sh +++ b/tests/fm-on.test.sh @@ -39,6 +39,11 @@ cat > "$REMOTE_ROOT/bin/fm-probe-two.sh" <<'SH' printf 'home=%s\nroot=%s\n' "$FM_HOME" "$FM_ROOT_OVERRIDE" if [ -n "${TOP_SECRET:-}" ]; then printf 'secret=leaked\n'; else printf 'secret=absent\n'; fi SH +cat > "$REMOTE_ROOT/bin/fm-probe-path.sh" <<'SH' +#!/usr/bin/env bash +printf '%s\n' "$PATH" +SH +cp "$ROOT/bin/fm-remote-doctor.sh" "$REMOTE_ROOT/bin/fm-remote-doctor.sh" cat > "$REMOTE_ROOT/bin/fm-mutate.sh" <<'SH' #!/usr/bin/env bash printf 'mutation\n' >> "$1" @@ -125,6 +130,106 @@ assert_contains "$out" "root=$REMOTE_ROOT" "remote root was not explicit" assert_contains "$out" 'secret=absent' "the primary ambient environment crossed the transport" pass "the fixed entrypoint sets only its explicit environment" +# The child PATH is the entrypoint's own composition, so it is asserted on the +# PATH a real child receives rather than on the script that builds it. The +# expectation is rebuilt here from the documented contract - fixed head, the +# package-manager directories that exist on this host, fixed tail - so a host +# with nix, homebrew, or neither exercises both the include and omit directions. +ACCOUNT_HOME=$(unset HOME; CDPATH='' cd ~ && pwd -P) +ACCOUNT_USER=$(id -un) +OPTIONAL_DIRS=( + "$ACCOUNT_HOME/.nix-profile/bin" + "/etc/profiles/per-user/$ACCOUNT_USER/bin" + /run/current-system/sw/bin + /opt/homebrew/bin + /usr/local/bin +) +EXPECTED_PATH= +expect_dir() { + case ":$EXPECTED_PATH:" in *":$1:"*) return 0 ;; esac + EXPECTED_PATH="${EXPECTED_PATH:+$EXPECTED_PATH:}$1" +} +path_has() { case ":$1:" in *":$2:"*) return 0 ;; esac; return 1; } +expect_dir "$REMOTE_ROOT/bin" +expect_dir "$ACCOUNT_HOME/.local/bin" +for candidate in "${OPTIONAL_DIRS[@]}"; do + [ -d "$candidate" ] && expect_dir "$candidate" +done +for fixed in /usr/bin /bin /usr/sbin /sbin; do expect_dir "$fixed"; done + +CHILD_PATH=$(fm_on ios fm-probe-path.sh) +[ "$CHILD_PATH" = "$EXPECTED_PATH" ] \ + || fail "composed child PATH did not match the portable contract"$'\n'"expected: $EXPECTED_PATH"$'\n'"actual: $CHILD_PATH" +[ "${CHILD_PATH%%:*}" = "$REMOTE_ROOT/bin" ] || fail "the remote code root's bin was not first on the child PATH" +[ "$(printf '%s' "$CHILD_PATH" | cut -d: -f2)" = "$ACCOUNT_HOME/.local/bin" ] \ + || fail "the account's ~/.local/bin was not second on the child PATH" +case "$CHILD_PATH" in *:/usr/bin:/bin:/usr/sbin:/sbin) ;; *) fail "the child PATH did not end with the portable system tail" ;; esac +DUPES=$(printf '%s\n' "$CHILD_PATH" | tr ':' '\n' | sort | uniq -d) +[ -z "$DUPES" ] || fail "the child PATH repeated entries: $DUPES" +PRESENT_CHECKED=0 +ABSENT_CHECKED=0 +for candidate in "${OPTIONAL_DIRS[@]}"; do + if [ -d "$candidate" ]; then + path_has "$CHILD_PATH" "$candidate" || fail "an existing package-manager directory was dropped: $candidate" + PRESENT_CHECKED=$((PRESENT_CHECKED + 1)) + else + path_has "$CHILD_PATH" "$candidate" && fail "an absent directory was added to the child PATH: $candidate" + ABSENT_CHECKED=$((ABSENT_CHECKED + 1)) + fi +done +pass "the entrypoint composes a deduplicated child PATH (kept $PRESENT_CHECKED existing, omitted $ABSENT_CHECKED absent)" + +set +e +out=$( + # The entrypoint's subprocess invokes this indirectly through export -f. + # shellcheck disable=SC2329 + command() { + if [ "${1:-}" = -v ] && [ "${2:-}" = git ]; then return 1; fi + builtin command "$@" + } + if command -v git >/dev/null 2>&1; then + fail "the missing-git fixture still resolved git" + fi + export -f command + fm_on ios fm-remote-doctor.sh 2>&1 +) +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "the entrypoint passed when git did not resolve on its operator PATH" +assert_contains "$out" 'required tool git does not resolve on the remote operator PATH' "the entrypoint did not name the missing prerequisite" +assert_contains "$out" '/.local/bin' "the entrypoint did not point at the wrapper escape hatch" +assert_not_contains "$out" 'command is not tracked by the configured remote root' "missing git was misreported as an untracked command" +pass "the entrypoint gives an actionable missing-git diagnostic" + +out=$(fm_on ios fm-remote-doctor.sh) +rc=$? +expect_code 0 "$rc" "the remote doctor failed through the transport" +assert_contains "$out" "path=$EXPECTED_PATH" "the remote doctor did not report the entrypoint child PATH" +assert_contains "$out" 'entrypoint=yes' "the remote doctor did not detect its entrypoint launch" +assert_contains "$out" 'required git=' "the remote doctor did not report the required tool" +pass "the remote doctor reports the same PATH the entrypoint hands its children" + +DOCTOR_BIN="$TMP_ROOT/doctor-bin" +mkdir -p "$DOCTOR_BIN" +ln -sf "$(command -v bash)" "$DOCTOR_BIN/bash" +set +e +out=$(PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) +rc=$? +set -e +[ "$rc" -ne 0 ] || fail "the remote doctor passed with a missing required tool" +assert_contains "$out" 'required git=MISSING' "the remote doctor did not mark the missing required tool" +assert_contains "$out" 'required tools do not resolve on the remote runtime PATH: git' "the remote doctor did not name the missing tool" +assert_contains "$out" '.local/bin' "the remote doctor did not offer the wrapper escape hatch" +ln -sf "$(command -v git)" "$DOCTOR_BIN/git" +set +e +out=$(PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) +rc=$? +set -e +expect_code 0 "$rc" "the remote doctor failed with every required tool present" +assert_contains "$out" "required git=$DOCTOR_BIN/git" "the remote doctor did not report where the required tool resolved" +assert_contains "$out" 'optional tmux=absent' "the remote doctor did not report an absent optional tool" +pass "the remote doctor fails only on missing required tools and names them" + out=$(fm_on ios fm-probe-two.sh) assert_contains "$out" "home=$REMOTE_HOME" "first dynamic command stopped resolving" ARGV_TWO="$REMOTE_HOME/argv-two.bin" @@ -147,9 +252,30 @@ cat > "$REMOTE_ROOT/bin/fm-untracked.sh" <<'SH' printf 'untracked command ran\n' SH chmod +x "$REMOTE_ROOT/bin/fm-untracked.sh" -if fm_on ios fm-untracked.sh >/dev/null 2>&1; then +GIT_SHADOW_LOG="$TMP_ROOT/git-shadow.log" +cat > "$REMOTE_ROOT/bin/git" <<'SH' +#!/usr/bin/env bash +printf 'consulted\n' >> "$FM_GIT_SHADOW_LOG" +exit 0 +SH +chmod +x "$REMOTE_ROOT/bin/git" +FM_GIT_SHADOW_LOG="$GIT_SHADOW_LOG" "$REMOTE_ROOT/bin/git" -C "$REMOTE_ROOT" ls-files --error-unmatch bin/fm-untracked.sh \ + || fail "the checkout-local git shim did not demonstrate that it would authorize the untracked command" +untracked_root_b64=$(printf '%s' "$REMOTE_ROOT" | base64 | tr -d '\n') +untracked_home_b64=$(printf '%s' "$REMOTE_HOME" | base64 | tr -d '\n') +untracked_argv_b64=$(printf '%s\0' fm-untracked.sh | base64 | tr -d '\n') +set +e +out=$(FM_GIT_SHADOW_LOG="$GIT_SHADOW_LOG" "$REMOTE_ROOT/bin/fm-remote-entrypoint.sh" \ + 1 "$untracked_root_b64" "$untracked_home_b64" "$untracked_argv_b64" 2>&1) +rc=$? +set -e +if [ "$rc" -eq 0 ]; then fail "an untracked fm-*.sh executable was accepted" fi +assert_contains "$out" 'command is not tracked by the configured remote root' "the untracked command did not fail at tracked-command authorization" +[ "$(wc -l < "$GIT_SHADOW_LOG" | tr -d ' ')" -eq 1 ] \ + || fail "the tracked-command authorization consulted checkout-local git" +pass "tracked-command authorization excludes checkout-local git" if FM_HOME="$LOCAL_HOME" FM_ROOT_OVERRIDE="$REMOTE_ROOT" FM_SSH_BIN="$FAKEBIN/fake-ssh" \ "$ROOT/bin/fm-on.sh" '-oProxyCommand=bad' fm-probe-two.sh >/dev/null 2>&1; then fail "an option-shaped SSH route was accepted" diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index 1853fd4c063..a66e441fe80 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -132,6 +132,11 @@ case "${FM_FAKE_SSH_MODE:-normal}:$command_name:$command_rel" in "$FM_FAKE_REMOTE_ENTRYPOINT" "$@" < "$FM_FAKE_INHERIT_PAYLOAD" exit $? ;; + doctor-fail:fm-remote-doctor.sh:*) + printf 'required git=MISSING\n' + printf 'error: required tools do not resolve on the remote runtime PATH: git\n' >&2 + exit 1 + ;; provision-block-fail:fm-remote-home-provision.sh:*) touch "$FM_FAKE_SEED_ENTERED" while [ ! -f "$FM_FAKE_SEED_RELEASE" ]; do sleep 0.02; done @@ -277,6 +282,25 @@ assert_grep '- seed-keep ' "$TMP_ROOT/seed-parent/data/secondmates.md" "failed s assert_present "$TMP_ROOT/seed-keep-home/.fm-secondmate-home" "serialized seed lost its published remote home" pass "remote seed rollback preserves serialized competing routes" +# A remote that cannot run the basic toolchain must be rejected by the preflight +# before any home is created on that host. +if FM_SECONDMATE_CHARTER='Toolless host charter.' FM_SECONDMATE_SCOPE='toolless host' \ + FM_FAKE_SSH_MODE=doctor-fail seed_env "$ROOT/bin/fm-remote-home-seed.sh" \ + seed-toolless remote-mac "$REMOTE_ROOT" "$TMP_ROOT/seed-toolless-home" --no-projects \ + > "$TMP_ROOT/seed-toolless.out" 2>&1; then + fail "seeding proceeded against a remote that cannot run the required tools" +fi +assert_grep 'required tools do not resolve on the remote runtime PATH: git' \ + "$TMP_ROOT/seed-toolless.out" "the seed hid the remote runtime diagnostics" +assert_grep 'remote runtime preflight failed' "$TMP_ROOT/seed-toolless.out" \ + "the seed did not report the failing stage" +assert_absent "$TMP_ROOT/seed-toolless-home" "the seed provisioned a home despite a failing preflight" +assert_no_grep '- seed-toolless ' "$TMP_ROOT/seed-parent/data/secondmates.md" \ + "the refused route survived the preflight rollback" +assert_absent "$TMP_ROOT/seed-parent/data/seed-toolless/brief.md" \ + "the refused route left its scaffolded charter behind" +pass "remote seeding stops on the runtime preflight before touching the host" + # Provision and register the remote route from the captain-facing primary. out=$(FM_SECONDMATE_CHARTER='Own iOS delivery on the build Mac.' \ FM_SECONDMATE_SCOPE='iOS implementation and Xcode validation' \ From e5e8a671712bb8fbc3930ca0fcd182131c2a5637 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Mon, 3 Aug 2026 21:00:34 -0700 Subject: [PATCH 2/8] feat: gate remote second mates on Herdr readiness (#1639) * feat(bin): gate remote second mates on herdr readiness A remote second mate now always runs on the Herdr backend, whose server belongs to the host's GUI login session and therefore outlives the SSH connections that supervise it. fm-spawn's remote route forces that backend and the host-local control script refuses any other, so the requirement cannot be dropped from either side. fm-remote-doctor.sh becomes the single owner of what "ready" means. It keeps its PATH and tool reporting from #1623 and adds the Herdr, Aqua LaunchAgent, GUI-session, server-reachability, and entrypoint-symlink checks, tagging each gap fixable: or human: with the exact operator step. --fix closes only the automatable gaps - writing and loading the Aqua-scoped dev.firstmate.herdr launch agent, starting the server where no launch agent applies, and recreating the entrypoint symlink - then re-derives every check from the host, so a human gap is never presented as fixed. It never creates a login session, writes an auto-login password, or touches FileVault. Remote seed, remote spawn, and the startup liveness relaunch all run the same check, repair, re-check sequence through one shared library and fail closed with the doctor's own gap text. Recovery inherits the gate because it respawns through the same route. Tests drive the real doctor against a controlled account fixture with a private HOME, a state-backed launchctl, and a fake herdr, and prove the dangerous actions are never attempted. The remote lifecycle suites gain a stateful Herdr CLI fixture and answer the readiness gate at the SSH boundary, so they never inspect or repair the runner's own account. * no-mistakes(review): Validate launch-agent contract and confirm Herdr startup * no-mistakes(review): Validate loaded launch-agent contract before readiness * no-mistakes(review): Refuse legacy remote backends without altering routes * no-mistakes(review): Clarify conditional remote readiness repair sequence * no-mistakes(review): Repair remote readiness before liveness probing * no-mistakes(review): Preserve unknown seeds and reject legacy liveness * no-mistakes(document): docs: clarify remote Herdr backend ownership --- .../skills/secondmate-provisioning/SKILL.md | 2 +- bin/fm-bootstrap.sh | 36 +- bin/fm-remote-doctor.sh | 514 +++++++++++++++++- bin/fm-remote-home-seed.sh | 27 +- bin/fm-remote-readiness-lib.sh | 44 ++ bin/fm-remote-secondmate-control.sh | 41 +- bin/fm-spawn.sh | 46 +- bin/fm-test-run.sh | 1 + docs/architecture.md | 2 +- docs/herdr-backend.md | 1 + docs/remote-secondmates.md | 57 +- docs/scripts.md | 3 +- docs/verification/trace-context.md | 2 +- tests/fm-on.test.sh | 36 +- tests/fm-remote-doctor.test.sh | 438 +++++++++++++++ ...fm-remote-secondmate-lifecycle-e2e.test.sh | 257 ++++++++- ...fm-remote-secondmate-trace-context.test.sh | 51 +- tests/remote-herdr-fixture.sh | 125 +++++ 18 files changed, 1582 insertions(+), 101 deletions(-) create mode 100644 bin/fm-remote-readiness-lib.sh create mode 100755 tests/fm-remote-doctor.test.sh create mode 100644 tests/remote-herdr-fixture.sh diff --git a/.agents/skills/secondmate-provisioning/SKILL.md b/.agents/skills/secondmate-provisioning/SKILL.md index d128438163c..6d263e38efe 100644 --- a/.agents/skills/secondmate-provisioning/SKILL.md +++ b/.agents/skills/secondmate-provisioning/SKILL.md @@ -34,7 +34,7 @@ Each registry entry stays concise and single-line: the summary is one sentence n Natural-language summary and `scope:` text may contain parentheses and semicolons; keep the generated `(home: ...; scope: ...; projects: ...; added ...)` suffix intact so operational consumers resolve its explicit field markers. The `home:` path points to the seeded home containing `data/charter.md`; no extra registry pointer field is needed. For a remote route, `host:` is an OpenSSH config alias and `root:` is that host's separate tracked Firstmate code root. -Host placement is independent from the remote home's ordinary local runtime backend. +A remote second-mate agent always runs on the Herdr backend and every seed, launch, and liveness relaunch first gates its host on `bin/fm-remote-doctor.sh` readiness, so an unready host refuses with that doctor's own gap text rather than half-creating a route; the workers that second mate supervises keep the home's ordinary backend selection. This release places whole secondmate homes remotely and never individual workers. [`docs/remote-secondmates.md`](../../../docs/remote-secondmates.md) owns current operator setup and transport behavior. The home-seeded `data/charter.md` is the sole owner of boilerplate idle-by-default behavior, the normal delegation lifecycle, and standard escalation contracts, so point to that charter rather than restating those contracts in the registry entry. diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index 3171b8e7c52..ed1f5164984 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -120,6 +120,8 @@ DATA="${FM_DATA_OVERRIDE:-$FM_HOME/data}" . "$SCRIPT_DIR/fm-x-lib.sh" # shellcheck source=bin/fm-backend.sh disable=SC1091 . "$SCRIPT_DIR/fm-backend.sh" +# shellcheck source=bin/fm-remote-readiness-lib.sh disable=SC1091 +. "$SCRIPT_DIR/fm-remote-readiness-lib.sh" fleet_sync_origin_backed_project_count() { local count proj @@ -500,7 +502,7 @@ secondmate_liveness_sweep() { # primary-only no-op there. Mid-session liveness remains explicitly out of # scope and requires a separate periodic signal. [ -d "$STATE" ] || return 0 - local meta id window harness backend target agent_state out cause remote_host remote_rc + local meta id window harness backend target agent_state out cause remote_host remote_rc readiness_reason route_out remote_backend SECONDMATE_RESPAWNED_IDS="" for meta in "$STATE"/*.meta; do [ -f "$meta" ] || continue @@ -511,6 +513,20 @@ secondmate_liveness_sweep() { harness=$(fm_meta_get "$meta" harness) remote_host=$(fm_meta_get "$meta" remote_host) if [ -n "$remote_host" ]; then + remote_rc=0 + fm_remote_readiness_ensure "$SCRIPT_DIR" "$id" || remote_rc=$? + if [ "$remote_rc" -eq 255 ]; then + echo "SECONDMATE_LIVENESS: secondmate $id: skipped: remote host unavailable or endpoint state unknown; route preserved on $remote_host" + continue + fi + if [ "$remote_rc" -ne 0 ]; then + readiness_reason=$(printf '%s\n' "$FM_REMOTE_READINESS_OUT" \ + | awk '/^check [^=]+=(fixable|human):|^action:|^error:/ { print; exit }') + [ -n "$readiness_reason" ] || readiness_reason=$(first_line "$FM_REMOTE_READINESS_OUT") + [ -n "$readiness_reason" ] || readiness_reason="unknown readiness failure" + echo "SECONDMATE_LIVENESS: secondmate $id: skipped: remote readiness failed on $remote_host: $readiness_reason" + continue + fi if out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh state "$id" 2>/dev/null); then remote_rc=0 else @@ -527,6 +543,24 @@ secondmate_liveness_sweep() { agent_state=$(printf '%s\n' "$out" | tail -1) case "$agent_state" in alive) + if route_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh route "$id" 2>/dev/null); then + remote_rc=0 + else + remote_rc=$? + fi + if [ "$remote_rc" -eq 255 ]; then + echo "SECONDMATE_LIVENESS: secondmate $id: skipped: remote host unavailable or endpoint route unknown; route preserved on $remote_host" + continue + fi + if [ "$remote_rc" -ne 0 ]; then + echo "SECONDMATE_LIVENESS: secondmate $id: skipped: alive remote endpoint route is unreadable on $remote_host; inspect and migrate or retire it explicitly" + continue + fi + remote_backend=$(printf '%s\n' "$route_out" | sed -n 's/^backend=//p' | tail -1) + if [ "$remote_backend" != herdr ]; then + echo "SECONDMATE_LIVENESS: secondmate $id: skipped: alive remote endpoint is recorded on backend '${remote_backend:-missing}'; migrate or retire it explicitly" + continue + fi [ "${FM_BOOTSTRAP_VERBOSE_FACTS:-0}" != 1 ] || echo "BOOTSTRAP_INFO: remote secondmate $id already live (host=$remote_host)" ;; dead|missing) diff --git a/bin/fm-remote-doctor.sh b/bin/fm-remote-doctor.sh index fad8d09d0d7..fc2ee785a1d 100755 --- a/bin/fm-remote-doctor.sh +++ b/bin/fm-remote-doctor.sh @@ -1,24 +1,485 @@ #!/usr/bin/env bash -# Report the runtime PATH and tool resolution of a Firstmate remote account. +# Check, and optionally repair, one remote account's second-mate readiness. # # Usage: -# bin/fm-on.sh fm-remote-doctor.sh +# bin/fm-on.sh fm-remote-doctor.sh [--fix] # -# Read-only preflight. Run it through fm-on.sh so the fixed remote entrypoint -# composes and exports the child PATH: this command reports the PATH it inherits -# instead of recomposing it, so the entrypoint stays the single owner of that -# ordering and the two can never drift. Required tools that do not resolve are -# listed and set a non-zero exit status; optional tools are reported either way -# and never fail the check. Nothing on the host is created or modified. +# Run it through fm-on.sh so the fixed remote entrypoint composes and exports +# the child PATH: this command reports the PATH it inherits instead of +# recomposing it, so the entrypoint stays the single owner of that ordering and +# the two can never drift. +# +# A remote second mate always runs on the Herdr backend, so readiness is more +# than tool resolution. herdr must resolve, its server must be reachable, and on +# macOS the Firstmate-owned launch agent dev.firstmate.herdr at +# ~/Library/LaunchAgents/dev.firstmate.herdr.plist must exist, carry +# LimitLoadToSessionType=Aqua, and be loaded into the console user's gui/ +# domain, so the server belongs to the GUI login session and survives logout and +# SSH disconnection. SSH cannot create an Aqua session, so a host with no GUI +# login is reported as a human gap rather than repaired. +# +# Line protocol, one fact per line, stable for script consumers: +# mode=check|fix +# path= +# entrypoint=yes|no +# platform=darwin|linux||unknown +# required =|MISSING +# optional =|absent +# fix =applied: (--fix only) +# fix =failed: (--fix only) +# check =ok: +# check =skip: +# check =fixable: +# check =human: +# action: : +# Every check line is authoritative for the moment it printed: under --fix it is +# the state after the repair attempt, so a human gap is never presented as +# fixed. Any remaining fixable or human gap, and any missing required tool, +# exits non-zero. +# +# --fix is idempotent and closes only automatable gaps: it writes the Aqua +# launch agent, bootstraps and kickstarts it into gui/ when a login session +# exists, starts the herdr server where no launch agent applies, and recreates +# the entrypoint symlink. It never creates a login session, never writes an +# auto-login password (kcpassword), never changes FileVault, and never stores an +# account password; those remain reported human gaps. set -eu -REQUIRED_TOOLS=(git) -OPTIONAL_TOOLS=(tmux treehouse no-mistakes tasks-axi herdr claude codex opencode pi grok kimi) +# Resolve this script's directory with builtins only: a host missing a required +# tool must still reach the report that names it, not die on a bare PATH. +SCRIPT_SELF=${BASH_SOURCE[0]} +SCRIPT_DIR=${SCRIPT_SELF%/*} +[ "$SCRIPT_DIR" != "$SCRIPT_SELF" ] || SCRIPT_DIR=. +SCRIPT_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR" && pwd -P) +REQUIRED_TOOLS=(git jq) +OPTIONAL_TOOLS=(tmux treehouse no-mistakes tasks-axi claude codex opencode pi grok kimi) +LAUNCH_AGENT_LABEL=dev.firstmate.herdr +HERDR_SESSION_NAME=default +LAUNCH_AGENT_DIR="${HOME:-}/Library/LaunchAgents" +LAUNCH_AGENT_PLIST="$LAUNCH_AGENT_DIR/$LAUNCH_AGENT_LABEL.plist" +LAUNCH_AGENT_LOG_DIR="${HOME:-}/Library/Logs" +LAUNCH_AGENT_LOG="$LAUNCH_AGENT_LOG_DIR/$LAUNCH_AGENT_LABEL.log" +ENTRYPOINT_LINK="${HOME:-}/.local/bin/fm-remote-entrypoint.sh" -usage() { sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } +usage() { sed -n '2,5p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } +MODE=check +case "${1:-}" in + '') ;; + --fix) MODE=fix; shift ;; + *) usage ;; +esac [ "$#" -eq 0 ] || usage +PLATFORM_RAW=$(uname -s 2>/dev/null) || PLATFORM_RAW= +case "$PLATFORM_RAW" in + Darwin) PLATFORM=darwin ;; + Linux) PLATFORM=linux ;; + '') PLATFORM=unknown ;; + *) PLATFORM=$PLATFORM_RAW ;; +esac +UID_NUM=$(id -u 2>/dev/null) || UID_NUM= + +CHECK_NAMES=() +CHECK_VALUES=() +CHECK_ACTIONS=() + +record() { # [operator-action] + CHECK_NAMES+=("$1") + CHECK_VALUES+=("$2") + CHECK_ACTIONS+=("${3:-}") +} + +check_value() { # ; prints the recorded value, empty when unrecorded + local i=0 + while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do + if [ "${CHECK_NAMES[$i]}" = "$1" ]; then + printf '%s' "${CHECK_VALUES[$i]}" + return 0 + fi + i=$((i + 1)) + done + return 1 +} + +check_is_ok() { # + case "$(check_value "$1" 2>/dev/null || true)" in ok:*) return 0 ;; esac + return 1 +} + +herdr_cli_available() { + command -v herdr >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 +} + +# The herdr adapter is the single owner of session-scoped herdr invocation and +# of starting a server, so read and start through it rather than restating +# either here. Sourced only when both tools resolve, so a bare host still +# reports its gaps instead of failing to load. +herdr_adapter_load() { + [ -z "${FM_REMOTE_DOCTOR_HERDR_LOADED:-}" ] || return 0 + herdr_cli_available || return 1 + [ -f "$SCRIPT_DIR/fm-backend.sh" ] && [ -f "$SCRIPT_DIR/backends/herdr.sh" ] || return 1 + # shellcheck source=bin/fm-backend.sh + . "$SCRIPT_DIR/fm-backend.sh" || return 1 + fm_backend_source herdr || return 1 + FM_REMOTE_DOCTOR_HERDR_LOADED=1 +} + +herdr_server_running() { + local running + herdr_adapter_load || return 1 + running=$(fm_backend_herdr_cli "$HERDR_SESSION_NAME" status --json 2>/dev/null \ + | jq -r '.server.running // false' 2>/dev/null) || return 1 + [ "$running" = true ] +} + +launch_agent_is_aqua() { + local stripped + [ -f "$LAUNCH_AGENT_PLIST" ] && [ ! -L "$LAUNCH_AGENT_PLIST" ] || return 1 + stripped=$(tr -d ' \t\r\n' < "$LAUNCH_AGENT_PLIST" 2>/dev/null) || return 1 + case "$stripped" in + *'LimitLoadToSessionTypeAqua'*) return 0 ;; + esac + return 1 +} + +render_launch_agent() { # + local herdr_bin=$1 + cat < + + + + Label + $LAUNCH_AGENT_LABEL + ProgramArguments + + $herdr_bin + server + --session + $HERDR_SESSION_NAME + + LimitLoadToSessionType + Aqua + RunAtLoad + + KeepAlive + + StandardOutPath + $LAUNCH_AGENT_LOG + StandardErrorPath + $LAUNCH_AGENT_LOG + + +XML +} + +launch_agent_contract_matches() { + local herdr_bin actual expected + [ -f "$LAUNCH_AGENT_PLIST" ] && [ ! -L "$LAUNCH_AGENT_PLIST" ] || return 1 + herdr_bin=$(command -v herdr 2>/dev/null) || return 1 + actual=$(tr -d ' \t\r\n' < "$LAUNCH_AGENT_PLIST" 2>/dev/null) || return 1 + expected=$(render_launch_agent "$herdr_bin" | tr -d ' \t\r\n') || return 1 + [ "$actual" = "$expected" ] +} + +launch_agent_loaded_contract_matches() { + local loaded herdr_bin herdr_compact plist_compact log_compact args + herdr_bin=$(command -v herdr 2>/dev/null) || return 1 + loaded=$(launchctl print "gui/$UID_NUM/$LAUNCH_AGENT_LABEL" 2>/dev/null) || return 1 + loaded=$(printf '%s' "$loaded" | tr -d ' \t\r\n') || return 1 + herdr_compact=$(printf '%s' "$herdr_bin" | tr -d ' \t\r\n') || return 1 + plist_compact=$(printf '%s' "$LAUNCH_AGENT_PLIST" | tr -d ' \t\r\n') || return 1 + log_compact=$(printf '%s' "$LAUNCH_AGENT_LOG" | tr -d ' \t\r\n') || return 1 + args="arguments={$herdr_compact"'server--session'"$HERDR_SESSION_NAME}" + [[ "$loaded" == *"path=$plist_compact"* ]] || return 1 + [[ "$loaded" == *"program=$herdr_compact"* ]] || return 1 + [[ "$loaded" == *"$args"* ]] || return 1 + [[ "$loaded" == *"stdoutpath=$log_compact"* ]] || return 1 + [[ "$loaded" == *"stderrpath=$log_compact"* ]] || return 1 + [[ "$loaded" == *'properties=keepalive|runatload'* ]] || return 1 +} + +# --- checks ----------------------------------------------------------------- + +check_herdr() { + local resolved + if resolved=$(command -v herdr 2>/dev/null); then + record herdr "ok: $resolved" + return 0 + fi + record herdr "human: the herdr CLI does not resolve on the remote runtime PATH" \ + "install herdr from https://herdr.dev on that account, or add a ~/.local/bin wrapper for it; a remote second mate always runs on the Herdr backend" +} + +check_gui_session() { + if [ "$PLATFORM" != darwin ]; then + record gui-session "skip: no Aqua login session applies on $PLATFORM" + return 0 + fi + if [ -z "$UID_NUM" ]; then + record gui-session "human: the account uid could not be read, so its login session cannot be inspected" \ + "run 'id -u' on that account and report the failure; Firstmate cannot address gui/ without it" + return 0 + fi + if ! command -v launchctl >/dev/null 2>&1; then + record gui-session "human: launchctl does not resolve, so the login session cannot be inspected" \ + "restore /bin/launchctl on that macOS account; without it no launch agent can be inspected or loaded" + return 0 + fi + if launchctl print "gui/$UID_NUM" >/dev/null 2>&1; then + record gui-session "ok: gui/$UID_NUM" + return 0 + fi + record gui-session "human: no Aqua login session exists for uid $UID_NUM" \ + "log that account in once at the console, and enable automatic login in System Settings > Users & Groups if the machine runs headless; SSH cannot create a GUI session, and Firstmate never writes an auto-login password or changes FileVault" +} + +check_launch_agent() { + if [ "$PLATFORM" != darwin ]; then + record launchagent "skip: launch agents apply only on darwin" + record launchagent-scope "skip: launch agents apply only on darwin" + record launchagent-loaded "skip: launch agents apply only on darwin" + return 0 + fi + if [ -f "$LAUNCH_AGENT_PLIST" ] && [ ! -L "$LAUNCH_AGENT_PLIST" ]; then + if launch_agent_contract_matches; then + record launchagent "ok: $LAUNCH_AGENT_PLIST matches the Firstmate-owned contract" + else + record launchagent "fixable: $LAUNCH_AGENT_PLIST does not match the current Firstmate-owned contract" \ + "rerun this command with --fix to rewrite its label, program arguments, session scope, restart policy, and log paths" + fi + if launch_agent_is_aqua; then + record launchagent-scope "ok: LimitLoadToSessionType=Aqua" + else + record launchagent-scope "fixable: $LAUNCH_AGENT_PLIST is not scoped to the Aqua login session" \ + "rerun this command with --fix to rewrite it with LimitLoadToSessionType=Aqua" + fi + else + record launchagent "fixable: no Firstmate herdr launch agent at $LAUNCH_AGENT_PLIST" \ + "rerun this command with --fix to install it" + record launchagent-scope "skip: no launch agent is installed yet" + fi + check_launch_agent_loaded +} + +check_launch_agent_loaded() { + if [ -z "$UID_NUM" ] || ! command -v launchctl >/dev/null 2>&1; then + record launchagent-loaded "human: the launch agent domain gui/ cannot be inspected on this account" \ + "restore launchctl and a readable account uid, then rerun this command" + return 0 + fi + if launchctl print "gui/$UID_NUM/$LAUNCH_AGENT_LABEL" >/dev/null 2>&1; then + if launch_agent_loaded_contract_matches; then + record launchagent-loaded "ok: gui/$UID_NUM/$LAUNCH_AGENT_LABEL matches the effective contract" + else + record launchagent-loaded "fixable: gui/$UID_NUM/$LAUNCH_AGENT_LABEL does not match the effective Firstmate-owned contract" \ + "rerun this command with --fix to replace the loaded job with the current launch-agent contract" + fi + return 0 + fi + if check_is_ok gui-session; then + record launchagent-loaded "fixable: $LAUNCH_AGENT_LABEL is not loaded into gui/$UID_NUM" \ + "rerun this command with --fix to bootstrap and start it" + return 0 + fi + record launchagent-loaded "human: $LAUNCH_AGENT_LABEL cannot be loaded because gui/$UID_NUM has no login session" \ + "close the login-session gap first; a launch agent can only be bootstrapped into an existing GUI session" +} + +check_herdr_server() { + if ! herdr_cli_available; then + record herdr-server "human: herdr server status cannot be read without both herdr and jq on the runtime PATH" \ + "install the missing tool reported above, then rerun this command" + return 0 + fi + if herdr_server_running; then + record herdr-server "ok: session $HERDR_SESSION_NAME is running" + return 0 + fi + if [ "$PLATFORM" = darwin ] && ! check_is_ok gui-session; then + record herdr-server "human: the herdr server for session $HERDR_SESSION_NAME is not running and there is no GUI login session to start it in" \ + "close the login-session gap first; a server started over SSH would not belong to an Aqua session" + return 0 + fi + record herdr-server "fixable: the herdr server for session $HERDR_SESSION_NAME is not running" \ + "rerun this command with --fix to start it" +} + +check_entrypoint_link() { + local want + if [ -z "${FM_ROOT_OVERRIDE:-}" ]; then + record entrypoint-link "skip: this run did not come through the fixed remote entrypoint" + return 0 + fi + want="$FM_ROOT_OVERRIDE/bin/fm-remote-entrypoint.sh" + if [ -L "$ENTRYPOINT_LINK" ] && [ "$(readlink "$ENTRYPOINT_LINK")" = "$want" ]; then + record entrypoint-link "ok: $ENTRYPOINT_LINK" + return 0 + fi + if [ -e "$ENTRYPOINT_LINK" ] || [ -L "$ENTRYPOINT_LINK" ]; then + record entrypoint-link "human: $ENTRYPOINT_LINK exists but is not the symlink to $want" \ + "inspect that path yourself and replace it with 'ln -sfn $want $ENTRYPOINT_LINK' if it is stale; Firstmate never overwrites a file it did not create there" + return 0 + fi + record entrypoint-link "fixable: no entrypoint symlink at $ENTRYPOINT_LINK" \ + "rerun this command with --fix to create it" +} + +run_checks() { + CHECK_NAMES=() + CHECK_VALUES=() + CHECK_ACTIONS=() + check_herdr + check_gui_session + check_launch_agent + check_herdr_server + check_entrypoint_link +} + +# --- repairs ---------------------------------------------------------------- + +fix_report() { # applied|failed + printf 'fix %s=%s: %s\n' "$1" "$2" "$3" +} + +write_launch_agent() { + local herdr_bin tmp + if ! herdr_bin=$(command -v herdr 2>/dev/null); then + fix_report launchagent failed "herdr does not resolve, so no launch agent was written" + return 1 + fi + case "$herdr_bin" in + *'&'*|*'<'*|*'>'*|*'"'*|*"'"*) + fix_report launchagent failed "the resolved herdr path contains characters that cannot be embedded in a property list: $herdr_bin" + return 1 + ;; + esac + if ! mkdir -p "$LAUNCH_AGENT_DIR" 2>/dev/null; then + fix_report launchagent failed "cannot create $LAUNCH_AGENT_DIR" + return 1 + fi + mkdir -p "$LAUNCH_AGENT_LOG_DIR" 2>/dev/null || true + tmp="$LAUNCH_AGENT_DIR/.$LAUNCH_AGENT_LABEL.plist.tmp.$$" + render_launch_agent "$herdr_bin" > "$tmp" + chmod 0644 "$tmp" 2>/dev/null || true + if ! mv -f -- "$tmp" "$LAUNCH_AGENT_PLIST" 2>/dev/null; then + rm -f -- "$tmp" + fix_report launchagent failed "cannot publish $LAUNCH_AGENT_PLIST" + return 1 + fi + fix_report launchagent applied "wrote the Aqua-scoped $LAUNCH_AGENT_LABEL launch agent running $herdr_bin server" +} + +# Reload rather than plain bootstrap so a rewritten plist replaces a stale +# in-memory copy, and kickstart so the server is running now rather than at the +# next login. Both are safe to repeat. +reload_launch_agent() { # + local report=$1 out + [ -f "$LAUNCH_AGENT_PLIST" ] || { + fix_report "$report" failed "there is no launch agent to load at $LAUNCH_AGENT_PLIST" + return 1 + } + if [ -z "$UID_NUM" ] || ! command -v launchctl >/dev/null 2>&1; then + fix_report "$report" failed "launchctl or the account uid is unavailable" + return 1 + fi + launchctl bootout "gui/$UID_NUM/$LAUNCH_AGENT_LABEL" >/dev/null 2>&1 || true + if ! out=$(launchctl bootstrap "gui/$UID_NUM" "$LAUNCH_AGENT_PLIST" 2>&1); then + fix_report "$report" failed "launchctl bootstrap gui/$UID_NUM refused: ${out:-no diagnostic}" + return 1 + fi + if ! out=$(launchctl kickstart -k "gui/$UID_NUM/$LAUNCH_AGENT_LABEL" 2>&1); then + fix_report "$report" failed "launchctl kickstart gui/$UID_NUM/$LAUNCH_AGENT_LABEL refused: ${out:-no diagnostic}" + return 1 + fi + if ! wait_for_herdr_server; then + fix_report "$report" failed "the herdr server for session $HERDR_SESSION_NAME did not report running within 10s" + return 1 + fi + fix_report "$report" applied "bootstrapped and started $LAUNCH_AGENT_LABEL in gui/$UID_NUM" +} + +wait_for_herdr_server() { + local i=0 + while [ "$i" -lt 20 ]; do + herdr_server_running && return 0 + i=$((i + 1)) + sleep 0.5 + done + return 1 +} + +start_herdr_server() { + if ! herdr_adapter_load; then + fix_report herdr-server failed "herdr and jq must both resolve before the server can be started" + return 1 + fi + if fm_backend_herdr_server_ensure "$HERDR_SESSION_NAME" >/dev/null 2>&1; then + fix_report herdr-server applied "started the herdr server for session $HERDR_SESSION_NAME" + return 0 + fi + fix_report herdr-server failed "the herdr server for session $HERDR_SESSION_NAME did not come up" + return 1 +} + +link_entrypoint() { + local want="${FM_ROOT_OVERRIDE:-}/bin/fm-remote-entrypoint.sh" + if ! mkdir -p "$(dirname "$ENTRYPOINT_LINK")" 2>/dev/null; then + fix_report entrypoint-link failed "cannot create $(dirname "$ENTRYPOINT_LINK")" + return 1 + fi + if ! ln -s "$want" "$ENTRYPOINT_LINK" 2>/dev/null; then + fix_report entrypoint-link failed "cannot create the symlink at $ENTRYPOINT_LINK" + return 1 + fi + fix_report entrypoint-link applied "linked $ENTRYPOINT_LINK to $want" +} + +apply_fixes() { + local i name value launch_agent_written=0 launch_agent_reloaded=0 + i=0 + while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do + name=${CHECK_NAMES[$i]} + value=${CHECK_VALUES[$i]} + i=$((i + 1)) + case "$value" in fixable:*) ;; *) continue ;; esac + case "$name" in + launchagent|launchagent-scope) + [ "$launch_agent_written" -eq 0 ] || continue + launch_agent_written=1 + write_launch_agent || continue + # A freshly written plist runs nothing until it is (re)loaded, and only + # an existing GUI session can hold it. + check_is_ok gui-session || continue + launch_agent_reloaded=1 + reload_launch_agent launchagent-loaded || true + ;; + launchagent-loaded) + [ "$launch_agent_reloaded" -eq 0 ] || continue + launch_agent_reloaded=1 + reload_launch_agent launchagent-loaded || true + ;; + herdr-server) + # On darwin the launch agent owns the server, so restart it through + # launchd rather than starting a stray one outside the Aqua session. A + # reload earlier in this same pass has already done that. + if [ "$PLATFORM" = darwin ] && [ -f "$LAUNCH_AGENT_PLIST" ] && check_is_ok gui-session; then + [ "$launch_agent_reloaded" -eq 0 ] || continue + launch_agent_reloaded=1 + reload_launch_agent herdr-server || true + continue + fi + start_herdr_server || true + ;; + entrypoint-link) link_entrypoint || true ;; + esac + done +} + +# --- report ----------------------------------------------------------------- + +printf 'mode=%s\n' "$MODE" printf 'path=%s\n' "${PATH:-}" if [ -n "${FM_ROOT_OVERRIDE:-}" ] && [ "${PATH%%:*}" = "$FM_ROOT_OVERRIDE/bin" ]; then printf 'entrypoint=yes\n' @@ -26,6 +487,7 @@ else printf 'entrypoint=no\n' printf 'note: not launched through the fixed remote entrypoint; the reported PATH is this caller environment.\n' >&2 fi +printf 'platform=%s\n' "$PLATFORM" MISSING=() for tool in "${REQUIRED_TOOLS[@]}"; do @@ -44,10 +506,38 @@ for tool in "${OPTIONAL_TOOLS[@]}"; do fi done +run_checks +if [ "$MODE" = fix ]; then + apply_fixes + # Re-derive every check from the host itself, so what prints below is the + # state after repair rather than the intent of a repair. + run_checks +fi + +GAPS=() +i=0 +while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do + printf 'check %s=%s\n' "${CHECK_NAMES[$i]}" "${CHECK_VALUES[$i]}" + case "${CHECK_VALUES[$i]}" in + fixable:*|human:*) GAPS+=("$i") ;; + esac + i=$((i + 1)) +done +for i in ${GAPS[@]+"${GAPS[@]}"}; do + [ -z "${CHECK_ACTIONS[$i]}" ] || printf 'action: %s: %s\n' "${CHECK_NAMES[$i]}" "${CHECK_ACTIONS[$i]}" +done + if [ "${#MISSING[@]}" -gt 0 ]; then printf 'error: required tools do not resolve on the remote runtime PATH: %s\n' "${MISSING[*]}" >&2 printf 'fix: install each one where it resolves on the path reported above, or put a wrapper script for it in %s/.local/bin, which is always on that PATH.\n' "${HOME:-~}" >&2 printf 'fix: tools provided by nvm, asdf, or mise never resolve here because no login or interactive shell runs; see docs/remote-secondmates.md for the wrapper recipe.\n' >&2 +fi +if [ "${#MISSING[@]}" -gt 0 ] || [ "${#GAPS[@]}" -gt 0 ]; then + NAMES= + for i in ${GAPS[@]+"${GAPS[@]}"}; do + NAMES="${NAMES:+$NAMES }${CHECK_NAMES[$i]}" + done + printf 'error: this host is not ready for a remote second mate%s\n' "${NAMES:+; unresolved: $NAMES}" >&2 exit 1 fi -printf 'ok: every required tool resolves on the remote runtime PATH\n' +printf 'ok: remote second-mate readiness confirmed on this host\n' diff --git a/bin/fm-remote-home-seed.sh b/bin/fm-remote-home-seed.sh index e0e723434ac..6288aa07e6e 100755 --- a/bin/fm-remote-home-seed.sh +++ b/bin/fm-remote-home-seed.sh @@ -6,12 +6,12 @@ # # The SSH alias must already reach a host whose non-interactive PATH exposes the # fixed fm-remote-entrypoint.sh from . The command records the -# remote host dimension in data/secondmates.md, preflights the remote runtime -# with fm-remote-doctor.sh before touching that host, sends a bounded provisioning +# remote host dimension in data/secondmates.md, gates the host on +# fm-remote-doctor.sh readiness before touching it, sends a bounded provisioning # manifest through fm-on.sh, and lets the remote host clone its own Firstmate # home and project origins. No project tree or secret environment is copied. # Known provisioning failure rolls the registry back. SSH status 255 preserves -# the route because completion is unknown and a same-route rerun converges. +# the route and any newly scaffolded brief because completion is unknown and a same-route rerun converges. set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -29,6 +29,8 @@ MAX_MANIFEST_BYTES=1048576 . "$SCRIPT_DIR/fm-secondmate-charter-lib.sh" # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-remote-readiness-lib.sh +. "$SCRIPT_DIR/fm-remote-readiness-lib.sh" die() { printf 'error: %s\n' "$1" >&2; exit 1; } usage() { sed -n '2,14p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } @@ -184,17 +186,22 @@ restore_registry_and_brief() { [ "$BRIEF_CREATED" -eq 0 ] || rm -f -- "$BRIEF" } -# Preflight the remote runtime before anything is created on that host. The -# doctor runs through the same fixed entrypoint as every later call, so it sees -# the exact PATH the remote home will run under. +# Preflight and, where it can, repair the remote runtime before anything is +# created on that host. The doctor runs through the same fixed entrypoint as +# every later call, so it sees the exact PATH the remote home will run under. set +e -PREFLIGHT_OUT=$("$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-doctor.sh 2>&1) +fm_remote_readiness_ensure "$SCRIPT_DIR" "$ID" PREFLIGHT_RC=$? set -e if [ "$PREFLIGHT_RC" -ne 0 ]; then - restore_registry_and_brief - [ -z "$PREFLIGHT_OUT" ] || printf '%s\n' "$PREFLIGHT_OUT" >&2 - die "remote runtime preflight failed; nothing was provisioned. Fix the reported tools, or update the remote code root if it predates fm-remote-doctor.sh" + if [ "$PREFLIGHT_RC" -ne 255 ]; then + restore_registry_and_brief + fi + [ -z "$FM_REMOTE_READINESS_OUT" ] || printf '%s\n' "$FM_REMOTE_READINESS_OUT" >&2 + if [ "$PREFLIGHT_RC" -eq 255 ]; then + die "remote readiness completion is unknown; route and brief preserved for same-host reconciliation" + fi + die "remote runtime preflight failed; nothing was provisioned. Close the gaps listed above, or update the remote code root if it predates the current fm-remote-doctor.sh" fi set +e diff --git a/bin/fm-remote-readiness-lib.sh b/bin/fm-remote-readiness-lib.sh new file mode 100644 index 00000000000..91184e2015b --- /dev/null +++ b/bin/fm-remote-readiness-lib.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# fm-remote-readiness-lib.sh - the remote second-mate readiness gate sequence. +# +# Source this file and call: +# fm_remote_readiness_ensure +# +# It runs bin/fm-remote-doctor.sh on that route's configured host, and when the +# read-only run reports any gap it runs the doctor again with --fix and then a +# third read-only time. That last read-only run is the verdict, so a repair is +# never trusted on its own word. bin/fm-remote-doctor.sh remains the single +# owner of every check, every repair, and every message; nothing here restates +# them. +# +# Returns 0 when the host is ready, 1 when a gap remains, and 255 when SSH could +# not complete. 255 means unknown remote completion, so a caller preserves its +# route and reconciles on the same host instead of treating it as a refusal. +# FM_REMOTE_READINESS_OUT always holds the output of the last run, which carries +# the check lines, the remaining human: gaps, and their exact operator actions. + +# Consumed by the sourcing caller, so every assignment reads as unused here. +# shellcheck disable=SC2034 +FM_REMOTE_READINESS_OUT= + +fm_remote_readiness_ensure() { # + local bin_dir=$1 id=$2 out rc + + out=$("$bin_dir/fm-on.sh" "$id" fm-remote-doctor.sh 2>&1) + rc=$? + FM_REMOTE_READINESS_OUT=$out + [ "$rc" -ne 0 ] || return 0 + [ "$rc" -ne 255 ] || return 255 + + out=$("$bin_dir/fm-on.sh" "$id" fm-remote-doctor.sh --fix 2>&1) + rc=$? + FM_REMOTE_READINESS_OUT=$out + [ "$rc" -ne 255 ] || return 255 + + out=$("$bin_dir/fm-on.sh" "$id" fm-remote-doctor.sh 2>&1) + rc=$? + FM_REMOTE_READINESS_OUT=$out + [ "$rc" -ne 255 ] || return 255 + [ "$rc" -eq 0 ] || return 1 + return 0 +} diff --git a/bin/fm-remote-secondmate-control.sh b/bin/fm-remote-secondmate-control.sh index 323c208fadf..9ba809eb32e 100755 --- a/bin/fm-remote-secondmate-control.sh +++ b/bin/fm-remote-secondmate-control.sh @@ -2,8 +2,9 @@ # Host-local lifecycle control for the remote secondmate home selected by fm-on. # # Usage: -# fm-remote-secondmate-control.sh launch [traceparent] +# fm-remote-secondmate-control.sh launch herdr [traceparent] # fm-remote-secondmate-control.sh state +# fm-remote-secondmate-control.sh route # fm-remote-secondmate-control.sh send # fm-remote-secondmate-control.sh key # fm-remote-secondmate-control.sh capture [lines] @@ -12,9 +13,12 @@ # fm-remote-secondmate-control.sh update # fm-remote-secondmate-control.sh retire [--force] # -# Remote placement ends here. The home still chooses its ordinary local runtime -# backend through its own config/backend, and fm-spawn/fm-send/fm-teardown keep -# owning those local endpoint mechanics. A private parent-route state directory +# Remote placement ends here, but the second-mate agent itself always runs on +# the Herdr backend, so launch refuses any other selection rather than reading +# this home's config/backend; fm-spawn/fm-send/fm-teardown keep owning the local +# endpoint mechanics, and the home's own workers keep its ordinary backend +# selection. bin/fm-remote-doctor.sh owns that host's readiness for Herdr, and +# docs/remote-secondmates.md owns why. A private parent-route state directory # stores only the remote secondmate agent's endpoint record; the home's own # state/*.meta remains reserved for workers the secondmate supervises. # @@ -38,7 +42,7 @@ CONTROL_DATA="$TARGET_HOME/data/.parent-route" . "$SCRIPT_DIR/fm-pending-reply-lib.sh" die() { printf 'error: %s\n' "$1" >&2; exit 1; } -usage() { sed -n '2,19p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } +usage() { sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'; exit 2; } validate_id() { case "$1" in ''|*[!A-Za-z0-9._-]*) die "invalid secondmate id: $1" ;; esac; } validate_home() { # [allow-absent] @@ -78,6 +82,17 @@ print_route() { # [ -z "$traceparent" ] || printf 'traceparent=%s\n' "$traceparent" } +cmd_route() { + local id=$1 meta + validate_id "$id" + validate_home "$id" + meta=$(meta_path "$id") + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + die "remote secondmate has no endpoint metadata" + fi + print_route "$id" +} + cmd_launch() { local id=$1 harness=$2 model=$3 effort=$4 selected_backend=$5 traceparent=${6:-} local current meta out backend target @@ -86,12 +101,22 @@ cmd_launch() { validate_home "$id" case "$harness" in claude|codex|opencode|pi|pi-signed|grok|kimi) ;; *) die "unverified remote secondmate harness: $harness" ;; esac case "$effort" in -|low|medium|high|xhigh|max) ;; *) die "invalid remote secondmate effort: $effort" ;; esac + # Herdr is required on this host, not merely preferred: its server belongs to + # the GUI login session, so the endpoint survives every SSH disconnection that + # a remote route depends on. bin/fm-remote-doctor.sh is the readiness owner. + case "$selected_backend" in herdr) ;; *) die "a remote secondmate runs only on the herdr backend, not '$selected_backend'" ;; esac mkdir -p "$CONTROL_STATE" "$CONTROL_DATA" meta=$(meta_path "$id") if [ -f "$meta" ]; then current=$(state_value "$id") case "$current" in - alive) print_route "$id"; return 0 ;; + alive) + backend=$(fm_backend_of_meta "$meta") + [ "$backend" = herdr ] \ + || die "remote secondmate $id has an alive endpoint recorded on backend '$backend'; refusing reuse until it is explicitly migrated or retired" + print_route "$id" + return 0 + ;; dead) backend=$(fm_backend_of_meta "$meta") target=$(fm_backend_target_of_meta "$meta") @@ -101,10 +126,9 @@ cmd_launch() { *) die "remote endpoint state is $current; refusing duplicate launch" ;; esac fi - ARGS=("$id" "$TARGET_HOME" --secondmate --harness "$harness") + ARGS=("$id" "$TARGET_HOME" --secondmate --harness "$harness" --backend "$selected_backend") [ "$model" = - ] || ARGS+=(--model "$model") [ "$effort" = - ] || ARGS+=(--effort "$effort") - [ "$selected_backend" = - ] || ARGS+=(--backend "$selected_backend") [ -z "$traceparent" ] || ARGS+=(--traceparent "$traceparent") if ! out=$(FM_HOME="$FM_ROOT" FM_ROOT_OVERRIDE="$FM_ROOT" \ FM_STATE_OVERRIDE="$CONTROL_STATE" FM_DATA_OVERRIDE="$CONTROL_DATA" \ @@ -239,6 +263,7 @@ cmd_retire() { case "${1:-}" in launch) shift; [ "$#" -ge 5 ] && [ "$#" -le 6 ] || usage; cmd_launch "$@" ;; state) shift; [ "$#" -eq 1 ] || usage; validate_id "$1"; validate_home "$1"; state_value "$1" ;; + route) shift; [ "$#" -eq 1 ] || usage; cmd_route "$1" ;; send) shift; [ "$#" -eq 2 ] || usage; cmd_send "$@" ;; key) shift; [ "$#" -eq 2 ] || usage; cmd_key "$@" ;; capture) shift; [ "$#" -ge 1 ] && [ "$#" -le 2 ] || usage; cmd_capture "$@" ;; diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index aff4f8eb44c..35a8df2b248 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -209,6 +209,8 @@ SUB_HOME_MARKER=".fm-secondmate-home" . "$SCRIPT_DIR/fm-pr-lib.sh" # shellcheck source=bin/fm-trace-context-lib.sh . "$SCRIPT_DIR/fm-trace-context-lib.sh" +# shellcheck source=bin/fm-remote-readiness-lib.sh +. "$SCRIPT_DIR/fm-remote-readiness-lib.sh" # Fail closed before any fleet mutation: a no-mistakes gate agent must never spawn # a direct report (see bin/fm-gate-refuse-lib.sh). fm_refuse_if_gate_agent @@ -394,7 +396,19 @@ spawn_remote_secondmate() { [ -n "$effort" ] || effort=- fi fi - backend=${BACKEND_ARG:--} + # A remote second mate always runs on Herdr: its server belongs to the host's + # own GUI login session, so the endpoint outlives every SSH connection that + # supervises it. bin/fm-remote-doctor.sh gates that host on the same + # requirement, and the remote home's config/backend never overrides it. + case "${BACKEND_ARG:--}" in + -|herdr) backend=herdr ;; + *) + fm_lock_release "$registry_lock" || true + fm_lock_release "$SPAWN_TASK_LOCK" || true + echo "error: a remote secondmate runs only on the herdr backend, not '$BACKEND_ARG'" >&2 + return 1 + ;; + esac case "$effort" in -|low|medium|high|xhigh|max) ;; *) @@ -417,6 +431,27 @@ spawn_remote_secondmate() { return 1 fi fi + # Gate the host before anything is published or transferred, so a host that + # cannot hold a durable Herdr endpoint refuses here rather than half-way + # through a launch. This is also the readiness gate every liveness relaunch + # passes through, because recovery respawns through this same route. + rc=0 + fm_remote_readiness_ensure "$SCRIPT_DIR" "$id" || rc=$? + if [ "$rc" -ne 0 ]; then + fm_lock_release "$registry_lock" || true + fm_lock_release "$SPAWN_TASK_LOCK" || true + # Summary first, then the doctor's own text: a caller that reports only the + # first line, such as the startup liveness sweep, must still say something + # actionable. + if [ "$rc" -eq 255 ]; then + echo "error: remote secondmate $id readiness could not be confirmed; preserved route $host:$home" >&2 + else + echo "error: remote secondmate $id host $host is not ready for a remote second mate; launch refused" >&2 + fi + [ -z "$FM_REMOTE_READINESS_OUT" ] || printf '%s\n' "$FM_REMOTE_READINESS_OUT" >&2 + [ "$rc" -ne 255 ] || return 255 + return 1 + fi remote_lock=$(fm_remote_inherit_transaction_lock_path "$STATE" "$id") if ! fm_lock_acquire_wait "$remote_lock"; then fm_lock_release "$registry_lock" || true @@ -478,7 +513,14 @@ spawn_remote_secondmate() { remote_backend=$(printf '%s\n' "$out" | sed -n 's/^backend=//p' | tail -1) remote_target=$(printf '%s\n' "$out" | sed -n 's/^target=//p' | tail -1) remote_harness=$(printf '%s\n' "$out" | sed -n 's/^harness=//p' | tail -1) - [ -n "$remote_backend" ] && [ -n "$remote_target" ] && [ "$remote_harness" = "$harness" ] || { + if [ "$remote_backend" != herdr ]; then + fm_lock_release "$remote_lock" || true + fm_lock_release "$registry_lock" || true + fm_lock_release "$SPAWN_TASK_LOCK" || true + echo "error: remote launch returned backend '${remote_backend:-missing}', expected herdr; preserving the remote route for reconciliation" >&2 + return 1 + fi + [ -n "$remote_target" ] && [ "$remote_harness" = "$harness" ] || { fm_lock_release "$remote_lock" || true fm_lock_release "$registry_lock" || true fm_lock_release "$SPAWN_TASK_LOCK" || true diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index eb3ae30151a..ad1cd905cd2 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -164,6 +164,7 @@ family_for_basename() { printf '%s\n' real-herdr-gated ;; fm-backlog-handoff.test.sh|fm-on.test.sh|fm-remote-backlog-handoff.test.sh|\ + fm-remote-doctor.test.sh|\ fm-remote-reply.test.sh|fm-remote-secondmate-lifecycle-e2e.test.sh|\ fm-remote-secondmate-trace-context.test.sh|\ fm-secondmate-harness.test.sh|fm-secondmate-lifecycle-e2e.test.sh|\ diff --git a/docs/architecture.md b/docs/architecture.md index e54a803ba94..879db1b2503 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -174,7 +174,7 @@ That keeps spawn launch compatible across claude, codex, grok, pi, opencode, and `data/secondmates.md` records persistent secondmates with natural-language scopes, project clone lists, and home paths. A local route points directly at its home, while a remote route adds an SSH alias and remote Firstmate code root so the entire home and all of its child work stay on that host. -Remote placement is independent of the remote home's ordinary local session backend, and individual workers are never placed remotely by this feature. +Remote placement pins the remote second-mate agent to Herdr while leaving the remote home's worker backend selection independent, and individual workers are never placed remotely by this feature. [`remote-secondmates.md`](remote-secondmates.md) owns current setup, transport, relay, failure, and retirement behavior. `fm-home-seed.sh` provisions a local isolated home, clones the listed PR-based projects into it, initializes newly cloned `no-mistakes` projects, copies the charter to `data/charter.md`, and `fm-spawn.sh --secondmate` launches it through the same session-provider and status-file path as any direct report. `fm-remote-home-seed.sh` sends a bounded charter and origin manifest through the generic transport so the remote host clones and provisions its own home and projects. diff --git a/docs/herdr-backend.md b/docs/herdr-backend.md index 83d527c588b..9834678df0d 100644 --- a/docs/herdr-backend.md +++ b/docs/herdr-backend.md @@ -20,6 +20,7 @@ Herdr is dual-licensed AGPL-3.0-or-later or commercial. Firstmate invokes its CLI as a separate process. Select Herdr with local `config/backend` containing `herdr`, `FM_BACKEND=herdr` for one launch, or an explicit request to Firstmate. +A remote second-mate agent is the one case with no choice: it always runs on Herdr, and [`remote-secondmates.md`](remote-secondmates.md) owns that requirement and the readiness its host must meet. It is also auto-detected when the primary runs natively under `HERDR_ENV=1` and is not inside tmux. A tmux pane nested inside Herdr resolves to tmux because the innermost multiplexer wins. An auto-detected Herdr spawn prints an opt-out notice. diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index bb87d82f94c..fa31953aba8 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -1,10 +1,13 @@ # Remote second mates Remote second mates place a whole persistent Firstmate home on another SSH-reachable host. -The primary still owns routing and supervision, while the remote home owns its own projects, backlog, workers, and local session backend. -Remote placement is a separate dimension from that host-local backend: a route can target one SSH host while the remote home uses tmux, Herdr, or another secondmate-capable backend. +The primary still owns routing and supervision, while the remote home owns its own projects, backlog, and workers. Firstmate does not support placing an individual worker remotely or failing a remote route over to a local replacement. +The remote second-mate agent itself always runs on the [Herdr backend](herdr-backend.md), and every path that provisions or launches one refuses a host that is not ready for it. +Herdr's server belongs to the host's own GUI login session rather than to the SSH connection, so the agent's endpoint survives every disconnection the primary's supervision depends on. +Local second mates are unaffected and keep their ordinary backend selection, as do the workers a remote second mate supervises inside its own home. + ## Prerequisites Configure an SSH alias in the primary account's normal OpenSSH configuration. @@ -57,15 +60,39 @@ Replace the placeholder with the remote account's selected nvm version. For asdf or mise, use the same shape with the selected version's absolute `bin` directory, one wrapper per tool the remote home actually needs. The wrapper must execute that absolute target rather than resolving its own name again through `~/.local/bin`. -Check any host against the real contract: +## Readiness, repair, and the human steps + +`bin/fm-remote-doctor.sh` is the single owner of what "ready for a remote second mate" means. +Check any host against it directly: ```sh bin/fm-on.sh fm-remote-doctor.sh ``` -The doctor is read-only. -It prints the exact `PATH` its own entrypoint launch produced, then reports where each required and optional tool resolved. -It exits non-zero and names every required tool that did not resolve, so the output is the install-or-shim list for that host. +That run is read-only. +It prints the exact `PATH` its own entrypoint launch produced, reports where each required and optional tool resolved, then reports one line per readiness check. +Each gap is tagged `fixable:` when `--fix` can close it or `human:` when only a person at that machine can, and every gap is followed by an `action:` line naming the exact step. +Any remaining gap exits non-zero. +The script's own header owns the full line protocol. + +`--fix` repairs only the automatable gaps and is safe to rerun: + +```sh +bin/fm-on.sh fm-remote-doctor.sh --fix +``` + +It writes and reloads the Firstmate-owned launch agent `dev.firstmate.herdr` at `~/Library/LaunchAgents/dev.firstmate.herdr.plist`, scoped with `LimitLoadToSessionType=Aqua` so it belongs to the GUI login session, bootstraps and starts it in `gui/`, starts the herdr server directly on a host where no launch agent applies, and recreates the `~/.local/bin/fm-remote-entrypoint.sh` symlink when it is absent. +It re-derives every check from the host afterwards, so what it prints is the state after the repair rather than the intent of one. + +These steps are never automated and are always reported rather than silently attempted, because SSH cannot create a GUI session from nothing: + +- The first console login on that Mac, and automatic login in System Settings > Users & Groups when the machine runs headless and must come back on its own after a reboot. +- FileVault, which holds a reboot at pre-boot authentication before any login session exists. +- Installing herdr from [herdr.dev](https://herdr.dev), and any required tool a wrapper cannot resolve. +- Each worker runtime's own `/login`, and any keychain password prompt that login needs. + +Firstmate never writes an auto-login password, never changes FileVault, and never stores an account password. +A file at `~/.local/bin/fm-remote-entrypoint.sh` that is not Firstmate's own symlink is reported for the operator to inspect and is never overwritten. ## Provision a route @@ -77,8 +104,9 @@ bin/fm-remote-home-seed.sh {` is the remote Firstmate code clone that supplies tracked scripts. `` is a separate absolute path for the persistent secondmate home and must not overlap the code root. -The seed records `host:`, `root:`, and `home:` in `data/secondmates.md`, preflights the host with `fm-remote-doctor.sh`, sends a bounded manifest, and lets the remote host clone its own Firstmate home and project origins. -A failing preflight prints the doctor's missing-tool list, restores the registry, and creates nothing on the remote host. +The seed records `host:`, `root:`, and `home:` in `data/secondmates.md`, gates the host on readiness, sends a bounded manifest, and lets the remote host clone its own Firstmate home and project origins. +Readiness starts with a read-only check; when that check reports a gap, it runs `--fix` and then a second read-only check whose verdict decides, so the operator never has to run the repair by hand and a repair is never trusted on its own word. +A host that stays red prints the doctor's remaining gaps and their operator steps, restores the registry, and creates nothing on the remote host. It does not copy project trees or the primary process environment. A known provisioning failure rolls back the new route, while SSH exit 255 preserves it because remote completion is unknown and must be reconciled on the same host. @@ -94,10 +122,14 @@ Launch or recover the remote second mate with the same command used for a local bin/fm-spawn.sh --secondmate ``` -The primary resolves the verified secondmate harness and optional model and effort, transfers the inherited-material allowlist, and asks the remote host to launch through that home's ordinary backend selection. +The primary resolves the verified secondmate harness and optional model and effort, runs the same readiness gate the seed runs, transfers the inherited-material allowlist, and asks the remote host to launch on Herdr. +An explicit request for any other backend is refused rather than honored, and the remote host refuses one too. +A launch after a host has drifted out of readiness fails with the doctor's own gap text instead of leaving a half-created endpoint. Raw launch commands are not accepted for remote secondmates. Backends that already refuse secondmate launch, currently Orca and cmux, remain unsupported on the remote host. +Startup liveness recovery relaunches a dead or missing remote second mate through this same command, so recovery passes the same readiness gate rather than a weaker one. + Send routed requests normally: ```sh @@ -153,15 +185,18 @@ No generic remote delete or write surface exists: remote writes are confined to ## Verification -The portable tests use the real entrypoint protocol, real git repositories, a deterministic SSH boundary, and a host-local backend fixture: +The portable tests use the real entrypoint protocol, real git repositories, a deterministic SSH boundary, a stateful host-local Herdr CLI fixture, and a controlled account fixture for the readiness gate: ```sh bin/fm-test-run.sh tests/fm-on.test.sh +bin/fm-test-run.sh tests/fm-remote-doctor.test.sh bin/fm-test-run.sh tests/fm-remote-reply.test.sh bin/fm-test-run.sh tests/fm-remote-backlog-handoff.test.sh bin/fm-test-run.sh tests/fm-remote-secondmate-lifecycle-e2e.test.sh bin/fm-test-run.sh tests/fm-remote-secondmate-trace-context.test.sh ``` -For a real-host smoke test, provision a disposable remote account and project, launch the second mate, send one marked request, verify its correlated reply and structured fleet projection, simulate an unreachable host to confirm unknown-without-failover behavior, then retire only after the remote queue is empty. +The account-level checks the doctor performs - a real Aqua login session, a real `launchctl` domain, and a real herdr server - are only ever exercised against fixtures here, so the readiness gate's behavior on a genuine Mac remains an operator-run smoke test. + +For a real-host smoke test, provision a disposable remote account and project, run the doctor and its repair against that account, launch the second mate, send one marked request, verify its correlated reply and structured fleet projection, simulate an unreachable host to confirm unknown-without-failover behavior, then retire only after the remote queue is empty. The deterministic suite is automated; real-host validation is still an operator-run smoke test and is not claimed by the repository tests. diff --git a/docs/scripts.md b/docs/scripts.md index 82e1e585d32..e59e02fe0a1 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -17,7 +17,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-bearings-snapshot.sh` | Project the fleet snapshot to the compact TOON bearings view; local-only unless `--include-prs` | | `fm-update.sh` | Fast-forward-only self-update of firstmate and local or remote secondmate homes | | `fm-on.sh` | Execute one tracked Firstmate command in a configured remote secondmate home | -| `fm-remote-doctor.sh` | Report the inherited remote runtime PATH and required or optional tool resolution | +| `fm-remote-doctor.sh` | Check, and with `--fix` repair, one remote account's second-mate readiness (Herdr, its Aqua launch agent, PATH, and tools) | | `fm-backlog-handoff.sh` | Validate and delegate queued backlog-item moves into a secondmate home | | `fm-backlog-receive.sh` | Idempotently ingest one confined remote handoff outbox through tasks-axi | | `fm-decision-hold.sh` | Create, verify, complete, and resolve durable captain-held decisions | @@ -42,6 +42,7 @@ The shared no-mistakes gate refusal for fleet lifecycle entrypoints is summarize | `fm-supervision-instructions.sh` | Render the session-start primary-harness supervision block or the one-line repair instruction | | `fm-home-seed.sh` | Transactionally provision a local secondmate home and maintain `data/secondmates.md` | | `fm-remote-home-seed.sh` | Register and provision a whole secondmate home on an SSH-reachable host | +| `fm-remote-readiness-lib.sh` | Shared remote second-mate readiness gate: check and, when needed, repair then re-check through `fm-remote-doctor.sh` | | `fm-spawn.sh` | Spawn crewmates, scouts, `id=repo` batches, and secondmates on the resolved harness and runtime backend | | `fm-backend.sh` | Runtime-backend selection, meta helpers, selector resolution, and operation dispatch | | `fm-backend-hometag-lib.sh` | Shared per-installation home-tag derivation for zellij tab and cmux workspace titles | diff --git a/docs/verification/trace-context.md b/docs/verification/trace-context.md index 0af75a4010e..c6af19f8d41 100644 --- a/docs/verification/trace-context.md +++ b/docs/verification/trace-context.md @@ -16,7 +16,7 @@ A final assertion drives the file-decided path (`FM_TRACE_CONTEXT` unset) and pr The suite touches no real harness or live fleet. `tests/fm-session-start.test.sh` additionally proves only a lock-owning session start writes the effective state and a lock-refused read-only start leaves it unchanged. -The remote-route suite `tests/fm-remote-secondmate-trace-context.test.sh` (6 assertions) covers the Secondmate path that never reaches the local export site, driving the real chain - the parent's `bin/fm-spawn.sh`, `bin/fm-on.sh`, the real remote entrypoint, `bin/fm-remote-secondmate-control.sh`, and the remote host's own `bin/fm-spawn.sh` - over the deterministic SSH boundary with a fake tmux, so the carrier the remote pane receives is read back from that pane's own log: disabled, the parent records no `traceparent=`, the remote pane receives no export, the remote home inherits no enablement flag, and the delivered snapshot is `FM_TRACE_CONTEXT=off` while `GOTMPDIR` still ships; enabled, the parent's recorded carrier, the remote endpoint's own record, and the exported pane value are one identical valid carrier sent after `GOTMPDIR` and before the launch command, with `FM_TRACE_CONTEXT=on` and the inherited flag delivered; a relaunch keeps that carrier verbatim in both the parent record and the pane export; a second remote route resolved from an environment holding a fixed ambient `TRACEPARENT` roots a trace id distinct from both that ambient carrier and the first route; the remote receiver accepts `config/trace-context` as ordinary declared inherited material while refusing `config/secondmate-harness`, which the primary deliberately does not propagate; and the delivery argument that carries a parent's carrier to a remote host is refused on a ship spawn, on a shell-metacharacter value, on an all-zero trace id, and on an empty value, so nothing but a strict W3C carrier on a Secondmate launch can reach a pane export. +The remote-route suite `tests/fm-remote-secondmate-trace-context.test.sh` (6 assertions) covers the Secondmate path that never reaches the local export site, driving the real chain - the parent's `bin/fm-spawn.sh`, `bin/fm-on.sh`, the real remote entrypoint, `bin/fm-remote-secondmate-control.sh`, and the remote host's own `bin/fm-spawn.sh` - over the deterministic SSH boundary with a stateful fake Herdr CLI, the backend a remote second mate always runs on, so the carrier the remote pane receives is read back from that pane's own log: disabled, the parent records no `traceparent=`, the remote pane receives no export, the remote home inherits no enablement flag, and the delivered snapshot is `FM_TRACE_CONTEXT=off` while `GOTMPDIR` still ships; enabled, the parent's recorded carrier, the remote endpoint's own record, and the exported pane value are one identical valid carrier sent after `GOTMPDIR` and before the launch command, with `FM_TRACE_CONTEXT=on` and the inherited flag delivered; a relaunch keeps that carrier verbatim in both the parent record and the pane export; a second remote route resolved from an environment holding a fixed ambient `TRACEPARENT` roots a trace id distinct from both that ambient carrier and the first route; the remote receiver accepts `config/trace-context` as ordinary declared inherited material while refusing `config/secondmate-harness`, which the primary deliberately does not propagate; and the delivery argument that carries a parent's carrier to a remote host is refused on a ship spawn, on a shell-metacharacter value, on an all-zero trace id, and on an empty value, so nothing but a strict W3C carrier on a Secondmate launch can reach a pane export. ```console $ bash tests/fm-trace-context-lib.test.sh | tail -1 diff --git a/tests/fm-on.test.sh b/tests/fm-on.test.sh index 3d041b2a681..2bd434a018a 100755 --- a/tests/fm-on.test.sh +++ b/tests/fm-on.test.sh @@ -201,34 +201,52 @@ assert_contains "$out" '/.local/bin' "the entrypoint did not point at the wrappe assert_not_contains "$out" 'command is not tracked by the configured remote root' "missing git was misreported as an untracked command" pass "the entrypoint gives an actionable missing-git diagnostic" -out=$(fm_on ios fm-remote-doctor.sh) -rc=$? -expect_code 0 "$rc" "the remote doctor failed through the transport" +# The doctor's readiness verdict depends on the host it runs on, which is this +# developer's or runner's real account here, so this transport test asserts only +# what the transport itself owns: the PATH the entrypoint handed the child. +# tests/fm-remote-doctor.test.sh owns the verdict against controlled fixtures. +set +e +out=$(fm_on ios fm-remote-doctor.sh 2>/dev/null) +set -e assert_contains "$out" "path=$EXPECTED_PATH" "the remote doctor did not report the entrypoint child PATH" assert_contains "$out" 'entrypoint=yes' "the remote doctor did not detect its entrypoint launch" assert_contains "$out" 'required git=' "the remote doctor did not report the required tool" pass "the remote doctor reports the same PATH the entrypoint hands its children" DOCTOR_BIN="$TMP_ROOT/doctor-bin" -mkdir -p "$DOCTOR_BIN" +DOCTOR_HOME="$TMP_ROOT/doctor-home" +mkdir -p "$DOCTOR_BIN" "$DOCTOR_HOME" ln -sf "$(command -v bash)" "$DOCTOR_BIN/bash" +# Report a non-darwin host so this file keeps testing tool resolution alone and +# never reads or writes the real account's launch agents. +cat > "$DOCTOR_BIN/uname" <<'SH' +#!/usr/bin/env bash +[ "${1:-}" = -s ] && { printf 'Linux\n'; exit 0; } +printf 'Linux\n' +SH +chmod +x "$DOCTOR_BIN/uname" set +e -out=$(PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) +out=$(HOME="$DOCTOR_HOME" PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) rc=$? set -e [ "$rc" -ne 0 ] || fail "the remote doctor passed with a missing required tool" assert_contains "$out" 'required git=MISSING' "the remote doctor did not mark the missing required tool" -assert_contains "$out" 'required tools do not resolve on the remote runtime PATH: git' "the remote doctor did not name the missing tool" +assert_contains "$out" 'required jq=MISSING' "the remote doctor did not mark every missing required tool" +assert_contains "$out" 'required tools do not resolve on the remote runtime PATH: git jq' "the remote doctor did not name the missing tools" assert_contains "$out" '.local/bin' "the remote doctor did not offer the wrapper escape hatch" ln -sf "$(command -v git)" "$DOCTOR_BIN/git" +# A resolvable stub is enough: this file asserts tool RESOLUTION, and with no +# herdr on the fixture PATH nothing ever asks jq to parse anything. +printf '#!/usr/bin/env bash\nexit 0\n' > "$DOCTOR_BIN/jq" +chmod +x "$DOCTOR_BIN/jq" set +e -out=$(PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) +out=$(HOME="$DOCTOR_HOME" PATH="$DOCTOR_BIN" "$ROOT/bin/fm-remote-doctor.sh" 2>&1) rc=$? set -e -expect_code 0 "$rc" "the remote doctor failed with every required tool present" assert_contains "$out" "required git=$DOCTOR_BIN/git" "the remote doctor did not report where the required tool resolved" assert_contains "$out" 'optional tmux=absent' "the remote doctor did not report an absent optional tool" -pass "the remote doctor fails only on missing required tools and names them" +assert_not_contains "$out" 'required tools do not resolve' "a resolved required tool was still reported missing" +pass "the remote doctor reports required and optional tool resolution and names what is missing" out=$(fm_on ios fm-probe-two.sh) assert_contains "$out" "home=$REMOTE_HOME" "first dynamic command stopped resolving" diff --git a/tests/fm-remote-doctor.test.sh b/tests/fm-remote-doctor.test.sh new file mode 100755 index 00000000000..6247846cb14 --- /dev/null +++ b/tests/fm-remote-doctor.test.sh @@ -0,0 +1,438 @@ +#!/usr/bin/env bash +# tests/fm-remote-doctor.test.sh - the remote second-mate readiness gate. +# +# Drives the real bin/fm-remote-doctor.sh against a controlled account fixture: +# a private HOME, a fake launchctl backed by state files, a fake herdr CLI, and +# a fake uname that selects the platform under test. Nothing here touches the +# runner's own launch agents, login session, or herdr server. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (the herdr adapter parses its JSON)"; exit 0; } + +TMP_ROOT=$(fm_test_tmproot fm-remote-doctor) +LABEL=dev.firstmate.herdr +CASE_N=0 + +# A fixture must be able to present a host with NO herdr, so the doctor never +# sees the runner's own PATH. Only the two required tools are re-exposed, by +# symlink, alongside the system directories the doctor's own helpers need. +TOOLS="$TMP_ROOT/tools" +mkdir -p "$TOOLS" +ln -sf "$(command -v git)" "$TOOLS/git" +ln -sf "$(command -v jq)" "$TOOLS/jq" +BASE_PATH="$TOOLS:/usr/bin:/bin:/usr/sbin:/sbin" + +# new_case [with-herdr] [gui] +# Builds one isolated account fixture and points the module-level CASE_* +# variables at it. "with-herdr" installs the fake herdr CLI; "gui" makes the +# fake launchctl report an existing Aqua login session. +new_case() { + local platform=$1 want_herdr=${2:-with-herdr} want_gui=${3:-gui} + CASE_N=$((CASE_N + 1)) + CASE_DIR="$TMP_ROOT/case$CASE_N" + CASE_BIN="$CASE_DIR/bin" + CASE_HOME="$CASE_DIR/home" + CASE_STATE="$CASE_DIR/state" + CASE_LAUNCHCTL_LOG="$CASE_STATE/launchctl.log" + CASE_FORBIDDEN_LOG="$CASE_STATE/forbidden.log" + CASE_HERDR_RUNNING="$CASE_STATE/herdr.running" + CASE_PLIST="$CASE_HOME/Library/LaunchAgents/$LABEL.plist" + mkdir -p "$CASE_BIN" "$CASE_HOME" "$CASE_STATE" + printf 'false\n' > "$CASE_HERDR_RUNNING" + : > "$CASE_LAUNCHCTL_LOG" + : > "$CASE_FORBIDDEN_LOG" + [ "$want_gui" != gui ] || touch "$CASE_STATE/gui-session" + + cat > "$CASE_BIN/uname" < "$CASE_BIN/launchctl" <<'SH' +#!/usr/bin/env bash +set -u +printf '%s\n' "$*" >> "$FM_FAKE_LAUNCHCTL_LOG" +domain=${2:-} +case "${1:-}" in + print) + case "$domain" in + */*/*) [ -f "$FM_FAKE_STATE/loaded-contract" ] || exit 113; cat "$FM_FAKE_STATE/loaded-contract" ;; + *) [ -f "$FM_FAKE_STATE/gui-session" ] || exit 113 ;; + esac + exit 0 + ;; + bootout) + [ ! -f "$FM_FAKE_STATE/bootout-fail" ] || { printf 'Boot-out failed: operation not permitted\n' >&2; exit 6; } + rm -f "$FM_FAKE_STATE/loaded-contract" + exit 0 + ;; + bootstrap) + # launchd refuses a gui/ domain that has no login session. + [ -f "$FM_FAKE_STATE/gui-session" ] || { printf 'Bootstrap failed: 5: Input/output error\n' >&2; exit 5; } + [ ! -f "$FM_FAKE_STATE/loaded-contract" ] || { printf 'Bootstrap failed: service already loaded\n' >&2; exit 5; } + cat > "$FM_FAKE_STATE/loaded-contract" < "$FM_FAKE_HERDR_RUNNING" + exit 0 + ;; + kickstart) + [ ! -f "$FM_FAKE_STATE/kickstart-fail" ] || { printf 'Kickstart failed: service unavailable\n' >&2; exit 6; } + if [ -f "$FM_FAKE_STATE/kickstart-delay" ]; then + cp "$FM_FAKE_STATE/kickstart-delay" "$FM_FAKE_STATE/herdr-delay" + else + printf 'true\n' > "$FM_FAKE_HERDR_RUNNING" + fi + exit 0 + ;; +esac +exit 0 +SH + + # Any attempt to reach for auto-login, FileVault, or the keychain records + # itself here so the test can prove the doctor never goes near them. + local forbidden + for forbidden in fdesetup security defaults; do + cat > "$CASE_BIN/$forbidden" <> "\$FM_FAKE_FORBIDDEN_LOG" +exit 0 +SH + chmod +x "$CASE_BIN/$forbidden" + done + + if [ "$want_herdr" = with-herdr ]; then + cat > "$CASE_BIN/herdr" <<'SH' +#!/usr/bin/env bash +set -u +running=$(cat "$FM_FAKE_HERDR_RUNNING" 2>/dev/null || printf 'false') +case "${1:-} ${2:-}" in + "status --json") + if [ -f "$FM_FAKE_STATE/herdr-delay" ]; then + delay=$(cat "$FM_FAKE_STATE/herdr-delay") + if [ "$delay" -gt 0 ]; then + printf '%s\n' "$((delay - 1))" > "$FM_FAKE_STATE/herdr-delay" + running=false + else + rm -f "$FM_FAKE_STATE/herdr-delay" + printf 'true\n' > "$FM_FAKE_HERDR_RUNNING" + running=true + fi + fi + printf '{"client":{"version":"0.7.5","protocol":16},"server":{"running":%s}}\n' "$running" + ;; + "server "*|"server ") + printf 'true\n' > "$FM_FAKE_HERDR_RUNNING" + ;; +esac +exit 0 +SH + chmod +x "$CASE_BIN/herdr" + fi + chmod +x "$CASE_BIN/uname" "$CASE_BIN/launchctl" + cat > "$CASE_BIN/sleep" <<'SH' +#!/usr/bin/env bash +exit 0 +SH + chmod +x "$CASE_BIN/sleep" +} + +# doctor [args...] -> runs the real doctor against the current fixture, +# capturing merged output in DOCTOR_OUT and its status in DOCTOR_RC. +doctor() { + set +e + DOCTOR_OUT=$( + HOME="$CASE_HOME" \ + PATH="$CASE_BIN:$BASE_PATH" \ + FM_FAKE_STATE="$CASE_STATE" \ + FM_FAKE_LAUNCHCTL_LOG="$CASE_LAUNCHCTL_LOG" \ + FM_FAKE_FORBIDDEN_LOG="$CASE_FORBIDDEN_LOG" \ + FM_FAKE_HERDR_RUNNING="$CASE_HERDR_RUNNING" \ + FM_FAKE_HERDR_BIN="$CASE_BIN/herdr" \ + FM_FAKE_PLIST="$CASE_PLIST" \ + FM_FAKE_LAUNCH_AGENT_LOG="$CASE_HOME/Library/Logs/$LABEL.log" \ + "$ROOT/bin/fm-remote-doctor.sh" "$@" 2>&1 + ) + DOCTOR_RC=$? + set -e +} + +write_loaded_contract() { # [properties] + local herdr_bin=$1 properties=${2:-'keepalive | runatload | inferred program'} + cat > "$CASE_STATE/loaded-contract" < + [ ! -s "$CASE_FORBIDDEN_LOG" ] \ + || fail "$1"$'\n'"--- attempted ---"$'\n'"$(cat "$CASE_FORBIDDEN_LOG")" + assert_absent "$CASE_HOME/Library/Preferences/com.apple.loginwindow.plist" \ + "the doctor wrote a loginwindow preference" + assert_absent "$CASE_HOME/kcpassword" "the doctor wrote an auto-login password" +} + +# --- a host with no herdr is never ready, and --fix cannot install one ------- + +new_case Darwin no-herdr gui +doctor +expect_code 1 "$DOCTOR_RC" "a host without herdr was reported ready" +assert_contains "$DOCTOR_OUT" 'check herdr=human:' "a missing herdr CLI was not tagged as a human gap" +assert_contains "$DOCTOR_OUT" 'action: herdr:' "a missing herdr CLI came with no operator action" +doctor --fix +expect_code 1 "$DOCTOR_RC" "--fix reported a host without herdr as ready" +assert_contains "$DOCTOR_OUT" 'check herdr=human:' "--fix stopped reporting the missing herdr CLI" +assert_not_contains "$DOCTOR_OUT" 'fix herdr=applied' "--fix claimed to have installed herdr" +assert_no_dangerous_calls "the doctor reached for auto-login, FileVault, or the keychain" +pass "a missing herdr CLI is a human gap that --fix never claims to close" + +# --- an absent launch agent is a fixable gap that --fix installs ------------- + +new_case Darwin with-herdr gui +doctor +expect_code 1 "$DOCTOR_RC" "a host with no launch agent was reported ready" +assert_contains "$DOCTOR_OUT" 'check herdr=ok:' "the fake herdr CLI was not detected" +assert_contains "$DOCTOR_OUT" 'check gui-session=ok:' "an existing login session was not detected" +assert_contains "$DOCTOR_OUT" 'check launchagent=fixable:' "an absent launch agent was not tagged fixable" +assert_contains "$DOCTOR_OUT" "$LABEL.plist" "the gap did not name the launch agent path" +assert_contains "$DOCTOR_OUT" 'check herdr-server=fixable:' "a stopped herdr server was not tagged fixable" +assert_absent "$CASE_PLIST" "a read-only doctor run installed a launch agent" +[ ! -s "$CASE_LAUNCHCTL_LOG" ] || assert_not_contains "$(cat "$CASE_LAUNCHCTL_LOG")" bootstrap \ + "a read-only doctor run loaded a launch agent" +pass "an absent launch agent is a fixable gap and the read-only run changes nothing" + +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix left a repairable host unready" +assert_contains "$DOCTOR_OUT" 'fix launchagent=applied:' "--fix did not report installing the launch agent" +assert_contains "$DOCTOR_OUT" 'check launchagent=ok:' "--fix did not re-check the installed launch agent" +assert_contains "$DOCTOR_OUT" 'check launchagent-scope=ok: LimitLoadToSessionType=Aqua' \ + "the installed launch agent was not Aqua-scoped" +assert_contains "$DOCTOR_OUT" 'check launchagent-loaded=ok:' "--fix did not load the launch agent" +assert_contains "$DOCTOR_OUT" 'check herdr-server=ok:' "--fix did not leave the herdr server running" +assert_present "$CASE_PLIST" "--fix reported success without writing the plist" +assert_grep 'Aqua' "$CASE_PLIST" "the written plist is not Aqua-scoped" +assert_grep "$LABEL" "$CASE_PLIST" "the written plist does not carry the Firstmate label" +assert_grep 'server' "$CASE_PLIST" "the written plist does not run a herdr server" +assert_grep "gui/$(id -u)" "$CASE_LAUNCHCTL_LOG" "the launch agent was not bootstrapped into the GUI domain" +assert_no_dangerous_calls "the repair reached for auto-login, FileVault, or the keychain" +pass "--fix installs, Aqua-scopes, loads, and starts the Firstmate herdr launch agent" + +PLIST_BEFORE=$(cat "$CASE_PLIST") +: > "$CASE_LAUNCHCTL_LOG" +doctor --fix +expect_code 0 "$DOCTOR_RC" "a second --fix on a ready host reported a gap" +assert_not_contains "$DOCTOR_OUT" 'fix launchagent=applied:' "a second --fix rewrote a healthy launch agent" +assert_not_contains "$DOCTOR_OUT" 'fix launchagent-loaded=applied:' "a second --fix reloaded a healthy launch agent" +[ "$(cat "$CASE_PLIST")" = "$PLIST_BEFORE" ] || fail "a second --fix changed the installed plist" +[ ! -s "$CASE_LAUNCHCTL_LOG" ] || assert_not_contains "$(cat "$CASE_LAUNCHCTL_LOG")" bootstrap \ + "a second --fix re-bootstrapped a loaded launch agent" +pass "--fix is idempotent once the host is ready" + +# --- a loaded, running launch agent with contract drift is repaired ---------- + +new_case Darwin with-herdr gui +mkdir -p "$(dirname "$CASE_PLIST")" +cat > "$CASE_PLIST" < + + + Label + $LABEL + ProgramArguments + + /obsolete/bin/herdr + server + --session + default + + LimitLoadToSessionType + Aqua + + +XML +write_loaded_contract /obsolete/bin/herdr 'runatload | inferred program' +printf 'true\n' > "$CASE_HERDR_RUNNING" +doctor +expect_code 1 "$DOCTOR_RC" "a stale launch-agent contract was reported ready" +assert_contains "$DOCTOR_OUT" 'check launchagent=fixable:' "launch-agent contract drift was not tagged fixable" +assert_contains "$DOCTOR_OUT" 'check launchagent-scope=ok:' "the independent Aqua scope was not recognized" +assert_contains "$DOCTOR_OUT" 'check launchagent-loaded=fixable:' "the stale effective launch-agent contract was not tagged fixable" +assert_contains "$DOCTOR_OUT" 'check herdr-server=ok:' "the running fixture was not recognized" +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix did not repair launch-agent contract drift" +assert_contains "$DOCTOR_OUT" 'check launchagent=ok:' "the repaired launch-agent contract was not confirmed" +assert_grep "$CASE_BIN/herdr" "$CASE_PLIST" "the repaired launch agent does not use the resolved herdr path" +assert_grep 'RunAtLoad' "$CASE_PLIST" "the repaired launch agent does not start at login" +assert_grep 'KeepAlive' "$CASE_PLIST" "the repaired launch agent is not kept alive" +assert_no_grep '/obsolete/bin/herdr' "$CASE_PLIST" "the obsolete herdr path survived repair" +pass "a loaded and running launch agent must match the complete owned contract" + +# --- failed replacement cannot hide a stale loaded launch-agent contract ----- + +new_case Darwin with-herdr gui +mkdir -p "$(dirname "$CASE_PLIST")" +cat > "$CASE_PLIST" < + + + Label + $LABEL + ProgramArguments + + /obsolete/bin/herdr + server + --session + default + + LimitLoadToSessionType + Aqua + + +XML +write_loaded_contract /obsolete/bin/herdr 'runatload | inferred program' +printf 'true\n' > "$CASE_HERDR_RUNNING" +touch "$CASE_STATE/bootout-fail" +doctor --fix +expect_code 1 "$DOCTOR_RC" "a stale loaded job passed after its replacement failed" +assert_contains "$DOCTOR_OUT" 'fix launchagent-loaded=failed: launchctl bootstrap' "the failed replacement was not reported" +assert_contains "$DOCTOR_OUT" 'check launchagent=ok:' "the repaired disk contract was not confirmed" +assert_contains "$DOCTOR_OUT" 'check launchagent-loaded=fixable:' "the stale loaded contract did not remain a readiness gap" +assert_contains "$DOCTOR_OUT" 'check herdr-server=ok:' "the existing server masking condition was not preserved" + +rm -f "$CASE_STATE/bootout-fail" +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix did not replace the stale loaded launch-agent contract" +assert_contains "$DOCTOR_OUT" 'check launchagent-loaded=ok:' "the replacement loaded contract was not confirmed" +assert_no_grep '/obsolete/bin/herdr' "$CASE_STATE/loaded-contract" "the stale effective program survived replacement" +pass "a failed reload leaves stale effective launch-agent state unready" + +# --- a launch agent that is not Aqua-scoped is repaired in place ------------- + +new_case Darwin with-herdr gui +mkdir -p "$(dirname "$CASE_PLIST")" +cat > "$CASE_PLIST" < + + + Label + $LABEL + LimitLoadToSessionType + Background + + +XML +doctor +expect_code 1 "$DOCTOR_RC" "a Background-scoped launch agent was reported ready" +assert_contains "$DOCTOR_OUT" 'check launchagent=fixable:' "an incomplete launch agent was not tagged fixable" +assert_contains "$DOCTOR_OUT" 'check launchagent-scope=fixable:' "a non-Aqua session scope was not tagged fixable" +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix could not re-scope an existing launch agent" +assert_contains "$DOCTOR_OUT" 'check launchagent-scope=ok: LimitLoadToSessionType=Aqua' \ + "--fix did not re-scope the launch agent to Aqua" +assert_no_grep 'Background' "$CASE_PLIST" "the Background session scope survived the repair" +pass "a launch agent outside the Aqua session scope is rewritten in place" + +# --- launchd start failures are reported and delayed readiness is awaited ---- + +new_case Darwin with-herdr gui +doctor --fix +expect_code 0 "$DOCTOR_RC" "the launch-agent startup fixture could not be initialized" +printf 'false\n' > "$CASE_HERDR_RUNNING" +touch "$CASE_STATE/bootstrap-does-not-start" "$CASE_STATE/kickstart-fail" +doctor --fix +expect_code 1 "$DOCTOR_RC" "a failed launchctl kickstart was reported ready" +assert_contains "$DOCTOR_OUT" 'fix herdr-server=failed: launchctl kickstart' "kickstart failure was not reported" +assert_contains "$DOCTOR_OUT" 'Kickstart failed: service unavailable' "kickstart diagnostic was discarded" +assert_not_contains "$DOCTOR_OUT" 'fix herdr-server=applied:' "a failed kickstart was reported as applied" +assert_contains "$DOCTOR_OUT" 'check herdr-server=fixable:' "the stopped server was not preserved as a readiness gap" + +rm -f "$CASE_STATE/kickstart-fail" +printf '2\n' > "$CASE_STATE/kickstart-delay" +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix did not wait for delayed launchd startup" +assert_contains "$DOCTOR_OUT" 'fix herdr-server=applied:' "delayed launchd startup was not reported as applied" +assert_contains "$DOCTOR_OUT" 'check herdr-server=ok:' "delayed launchd startup was not confirmed" +assert_absent "$CASE_STATE/herdr-delay" "the readiness poll stopped before the delayed server became reachable" +pass "launchd failures are reported and delayed server readiness is awaited" + +# --- no GUI login session: every dependent gap stays human ------------------- + +new_case Darwin with-herdr no-gui +doctor --fix +expect_code 1 "$DOCTOR_RC" "a host with no login session was reported ready" +assert_contains "$DOCTOR_OUT" 'check gui-session=human:' "an absent login session was not tagged human" +assert_contains "$DOCTOR_OUT" 'check launchagent-loaded=human:' "loading without a login session was not tagged human" +assert_contains "$DOCTOR_OUT" 'check herdr-server=human:' "starting a server without a login session was not tagged human" +assert_not_contains "$DOCTOR_OUT" 'fix gui-session=applied' "--fix claimed to have created a login session" +assert_not_contains "$DOCTOR_OUT" 'fix launchagent-loaded=applied' "--fix claimed to have loaded an unloadable launch agent" +assert_not_contains "$DOCTOR_OUT" 'fix herdr-server=applied' "--fix claimed to have started an unstartable server" +assert_contains "$DOCTOR_OUT" 'action: gui-session:' "the login-session gap came with no operator action" +assert_contains "$DOCTOR_OUT" 'automatic login' "the login-session action did not name the operator step" +assert_present "$CASE_PLIST" "--fix skipped the automatable launch-agent gap because a human gap existed" +assert_contains "$DOCTOR_OUT" 'error: this host is not ready for a remote second mate' \ + "a remaining human gap did not fail the readiness verdict" +assert_no_dangerous_calls "the doctor tried to create a login session by force" +pass "human gaps are reported with their operator step and never claimed as fixed" + +# --- linux has no launch agent, and --fix starts the server directly --------- + +new_case Linux with-herdr no-gui +doctor +expect_code 1 "$DOCTOR_RC" "a linux host with a stopped herdr server was reported ready" +assert_contains "$DOCTOR_OUT" 'platform=linux' "the platform was misreported" +assert_contains "$DOCTOR_OUT" 'check launchagent=skip:' "launch agents were checked on linux" +assert_contains "$DOCTOR_OUT" 'check gui-session=skip:' "an Aqua login session was required on linux" +assert_contains "$DOCTOR_OUT" 'check herdr-server=fixable:' "a stopped linux herdr server was not tagged fixable" +doctor --fix +expect_code 0 "$DOCTOR_RC" "--fix did not start the herdr server on linux" +assert_contains "$DOCTOR_OUT" 'fix herdr-server=applied:' "--fix did not report starting the server" +assert_contains "$DOCTOR_OUT" 'check herdr-server=ok:' "the started server was not confirmed by the re-check" +[ ! -s "$CASE_LAUNCHCTL_LOG" ] || fail "the linux path invoked launchctl" +pass "a non-darwin host skips launch agents and starts its herdr server directly" + +# --- the entrypoint symlink is recreated when it is missing ------------------ + +new_case Linux with-herdr no-gui +REMOTE_ROOT="$CASE_DIR/remote-root" +mkdir -p "$REMOTE_ROOT/bin" +printf '#!/usr/bin/env bash\n' > "$REMOTE_ROOT/bin/fm-remote-entrypoint.sh" +export FM_ROOT_OVERRIDE="$REMOTE_ROOT" +doctor +assert_contains "$DOCTOR_OUT" 'check entrypoint-link=fixable:' "a missing entrypoint symlink was not tagged fixable" +doctor --fix +assert_contains "$DOCTOR_OUT" 'fix entrypoint-link=applied:' "--fix did not report linking the entrypoint" +assert_contains "$DOCTOR_OUT" 'check entrypoint-link=ok:' "the recreated entrypoint symlink was not confirmed" +[ "$(readlink "$CASE_HOME/.local/bin/fm-remote-entrypoint.sh")" = "$REMOTE_ROOT/bin/fm-remote-entrypoint.sh" ] \ + || fail "the entrypoint symlink does not point at this code root" +printf 'not a symlink\n' > "$CASE_HOME/.local/bin/other" +rm -f "$CASE_HOME/.local/bin/fm-remote-entrypoint.sh" +printf 'operator wrapper\n' > "$CASE_HOME/.local/bin/fm-remote-entrypoint.sh" +doctor --fix +assert_contains "$DOCTOR_OUT" 'check entrypoint-link=human:' "an operator-owned entrypoint file was not left to the operator" +[ "$(cat "$CASE_HOME/.local/bin/fm-remote-entrypoint.sh")" = 'operator wrapper' ] \ + || fail "--fix overwrote a file it did not create" +unset FM_ROOT_OVERRIDE +pass "the entrypoint symlink is recreated when absent and never overwritten when operator-owned" diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index a66e441fe80..15e8d4e3c09 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -4,6 +4,8 @@ set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=tests/remote-herdr-fixture.sh +. "$(dirname "${BASH_SOURCE[0]}")/remote-herdr-fixture.sh" command -v jq >/dev/null 2>&1 || { echo "skip: jq not found"; exit 0; } ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P) @@ -16,6 +18,9 @@ REMOTE_HOME="$TMP_ROOT/remote-home" LOCAL_HOME="$TMP_ROOT/local-home" FAKEBIN=$(fm_fakebin "$TMP_ROOT/fake") SSH_COUNT="$TMP_ROOT/ssh.count" +DOCTOR_LOG="$TMP_ROOT/doctor.log" +HERDR_STATE="$TMP_ROOT/remote-herdr.state" +HERDR_LOG="$TMP_ROOT/remote-herdr.log" TMUX_LOG="$TMP_ROOT/remote-tmux.log" TMUX_STATE="$TMP_ROOT/remote-tmux.state" CLAIMS="$TMP_ROOT/claims" @@ -72,6 +77,8 @@ esac exit 0 SH chmod +x "$REMOTE_ROOT/bin/tmux" +install_remote_herdr_fixture "$REMOTE_ROOT" "$HERDR_STATE" "$HERDR_LOG" \ + "$TMP_ROOT/herdr-send-fail" "$TMP_ROOT/herdr.sock" git -C "$REMOTE_ROOT" init -q -b main git -C "$REMOTE_ROOT" config user.email test@example.com git -C "$REMOTE_ROOT" config user.name Test @@ -132,10 +139,64 @@ case "${FM_FAKE_SSH_MODE:-normal}:$command_name:$command_rel" in "$FM_FAKE_REMOTE_ENTRYPOINT" "$@" < "$FM_FAKE_INHERIT_PAYLOAD" exit $? ;; - doctor-fail:fm-remote-doctor.sh:*) - printf 'required git=MISSING\n' - printf 'error: required tools do not resolve on the remote runtime PATH: git\n' >&2 - exit 1 +esac +# The readiness gate is answered here rather than by the real doctor, which +# would inspect and repair the RUNNER's own account. tests/fm-remote-doctor.test.sh +# owns the doctor's real behavior against controlled account fixtures; this +# boundary owns only what the callers do with its verdict. +if [ "$command_name" = fm-remote-doctor.sh ]; then + printf '%s %s\n' "${FM_FAKE_SSH_MODE:-normal}" "${_command_action:--}" >> "$FM_FAKE_DOCTOR_LOG" + case "${FM_FAKE_SSH_MODE:-normal}" in + unreachable) exit 255 ;; + doctor-fix-unknown) + if [ "${_command_action:-}" = --fix ]; then + printf 'fix launchagent=applied: wrote the Aqua-scoped launch agent\n' + exit 255 + fi + printf 'check launchagent=fixable: no Firstmate herdr launch agent\n' + printf 'error: this host is not ready for a remote second mate; unresolved: launchagent\n' >&2 + exit 1 + ;; + doctor-human) + printf 'check gui-session=human: no Aqua login session exists for uid 501\n' + printf 'action: gui-session: log that account in once at the console\n' + printf 'error: this host is not ready for a remote second mate; unresolved: gui-session\n' >&2 + exit 1 + ;; + doctor-fixable) + # Red until --fix runs on this host, green on every later read-only run. + if [ "${_command_action:-}" = --fix ]; then + touch "$FM_FAKE_DOCTOR_REPAIRED" + printf 'fix launchagent=applied: wrote the Aqua-scoped launch agent\n' + printf 'ok: remote second-mate readiness confirmed on this host\n' + exit 0 + fi + [ -f "$FM_FAKE_DOCTOR_REPAIRED" ] || { + printf 'check launchagent=fixable: no Firstmate herdr launch agent\n' + printf 'error: this host is not ready for a remote second mate; unresolved: launchagent\n' >&2 + exit 1 + } + ;; + esac + printf 'check herdr=ok: /usr/bin/herdr\n' + printf 'ok: remote second-mate readiness confirmed on this host\n' + exit 0 +fi +if [ "${FM_FAKE_SSH_MODE:-normal}" = doctor-fixable ] \ + && [ "$command_name" = fm-remote-secondmate-control.sh ] \ + && [ "$_command_action" = state ] \ + && [ ! -f "$FM_FAKE_DOCTOR_REPAIRED" ]; then + printf 'unreadable\n' + exit 0 +fi +case "${FM_FAKE_SSH_MODE:-normal}:$command_name:$command_rel" in + launch-nonherdr-route:fm-remote-secondmate-control.sh:*) + [ "$_command_action" = launch ] || exit 93 + printf 'schema=fm-remote-secondmate-control.v1\n' + printf 'backend=tmux\n' + printf 'target=firstmate:fm-ios\n' + printf 'harness=codex\n' + exit 0 ;; provision-block-fail:fm-remote-home-provision.sh:*) touch "$FM_FAKE_SEED_ENTERED" @@ -179,6 +240,8 @@ remote_env() { FM_FAKE_REMOTE_CWD="$TMP_ROOT" \ FM_FAKE_SEED_ENTERED="$TMP_ROOT/seed.entered" \ FM_FAKE_SEED_RELEASE="$TMP_ROOT/seed.release" \ + FM_FAKE_DOCTOR_LOG="$DOCTOR_LOG" \ + FM_FAKE_DOCTOR_REPAIRED="$TMP_ROOT/doctor.repaired" \ FM_FAKE_INHERIT_ENTERED="$TMP_ROOT/inherit.entered" \ FM_FAKE_INHERIT_RELEASE="$TMP_ROOT/inherit.release" \ FM_FAKE_INHERIT_PAYLOAD="$TMP_ROOT/inherit.payload" \ @@ -202,6 +265,8 @@ seed_env() { FM_FAKE_REMOTE_CWD="$TMP_ROOT" \ FM_FAKE_SEED_ENTERED="$TMP_ROOT/seed.entered" \ FM_FAKE_SEED_RELEASE="$TMP_ROOT/seed.release" \ + FM_FAKE_DOCTOR_LOG="$DOCTOR_LOG" \ + FM_FAKE_DOCTOR_REPAIRED="$TMP_ROOT/doctor.repaired" \ "$@" } @@ -282,16 +347,39 @@ assert_grep '- seed-keep ' "$TMP_ROOT/seed-parent/data/secondmates.md" "failed s assert_present "$TMP_ROOT/seed-keep-home/.fm-secondmate-home" "serialized seed lost its published remote home" pass "remote seed rollback preserves serialized competing routes" -# A remote that cannot run the basic toolchain must be rejected by the preflight -# before any home is created on that host. -if FM_SECONDMATE_CHARTER='Toolless host charter.' FM_SECONDMATE_SCOPE='toolless host' \ - FM_FAKE_SSH_MODE=doctor-fail seed_env "$ROOT/bin/fm-remote-home-seed.sh" \ +: > "$DOCTOR_LOG" +if FM_SECONDMATE_CHARTER='Unknown readiness charter.' FM_SECONDMATE_SCOPE='unknown readiness' \ + FM_FAKE_SSH_MODE=doctor-fix-unknown seed_env "$ROOT/bin/fm-remote-home-seed.sh" \ + seed-unknown remote-mac "$REMOTE_ROOT" "$TMP_ROOT/seed-unknown-home" --no-projects \ + > "$TMP_ROOT/seed-unknown.out" 2>&1; then + fail "seeding claimed success after readiness repair completion became unknown" +fi +assert_grep 'remote readiness completion is unknown' "$TMP_ROOT/seed-unknown.out" \ + "unknown readiness did not report its distinct completion state" +assert_grep '- seed-unknown ' "$TMP_ROOT/seed-parent/data/secondmates.md" \ + "unknown readiness removed the registered route" +assert_present "$TMP_ROOT/seed-parent/data/seed-unknown/brief.md" \ + "unknown readiness removed the scaffolded brief" +assert_absent "$TMP_ROOT/seed-unknown-home" \ + "unknown readiness proceeded into remote home provisioning" +[ "$(cat "$DOCTOR_LOG")" = 'doctor-fix-unknown - +doctor-fix-unknown --fix' ] || fail "unknown readiness did not occur during the repair stage"$'\n'"$(cat "$DOCTOR_LOG")" +pass "unknown readiness preserves its route and brief for reconciliation" + +# A host that cannot hold a durable second mate must be rejected by the +# readiness gate before any home is created on it, and the operator must get the +# gap text rather than a bare refusal. +: > "$DOCTOR_LOG" +if FM_SECONDMATE_CHARTER='Unready host charter.' FM_SECONDMATE_SCOPE='unready host' \ + FM_FAKE_SSH_MODE=doctor-human seed_env "$ROOT/bin/fm-remote-home-seed.sh" \ seed-toolless remote-mac "$REMOTE_ROOT" "$TMP_ROOT/seed-toolless-home" --no-projects \ > "$TMP_ROOT/seed-toolless.out" 2>&1; then - fail "seeding proceeded against a remote that cannot run the required tools" + fail "seeding proceeded against a host that is not ready for a remote second mate" fi -assert_grep 'required tools do not resolve on the remote runtime PATH: git' \ - "$TMP_ROOT/seed-toolless.out" "the seed hid the remote runtime diagnostics" +assert_grep 'check gui-session=human:' \ + "$TMP_ROOT/seed-toolless.out" "the seed hid the remaining human gap" +assert_grep 'action: gui-session:' \ + "$TMP_ROOT/seed-toolless.out" "the seed hid the operator step that closes the gap" assert_grep 'remote runtime preflight failed' "$TMP_ROOT/seed-toolless.out" \ "the seed did not report the failing stage" assert_absent "$TMP_ROOT/seed-toolless-home" "the seed provisioned a home despite a failing preflight" @@ -299,7 +387,24 @@ assert_no_grep '- seed-toolless ' "$TMP_ROOT/seed-parent/data/secondmates.md" \ "the refused route survived the preflight rollback" assert_absent "$TMP_ROOT/seed-parent/data/seed-toolless/brief.md" \ "the refused route left its scaffolded charter behind" -pass "remote seeding stops on the runtime preflight before touching the host" +[ "$(cat "$DOCTOR_LOG")" = 'doctor-human - +doctor-human --fix +doctor-human -' ] || fail "the seed did not run the check, repair, re-check sequence"$'\n'"$(cat "$DOCTOR_LOG")" +pass "remote seeding checks, repairs, and re-checks readiness, then stops on a remaining gap" + +# The same gate must accept a host whose only gaps were repairable. +: > "$DOCTOR_LOG" +rm -f "$TMP_ROOT/doctor.repaired" +out=$(FM_SECONDMATE_CHARTER='Repairable host charter.' FM_SECONDMATE_SCOPE='repairable host' \ + FM_FAKE_SSH_MODE=doctor-fixable seed_env "$ROOT/bin/fm-remote-home-seed.sh" \ + seed-repair remote-mac "$REMOTE_ROOT" "$TMP_ROOT/seed-repair-home" --no-projects 2>&1) \ + || fail "seeding refused a host whose gaps the repair closed"$'\n'"$out" +assert_present "$TMP_ROOT/seed-repair-home/.fm-secondmate-home" "the repaired host was never provisioned" +assert_grep '- seed-repair ' "$TMP_ROOT/seed-parent/data/secondmates.md" "the repaired route was not registered" +[ "$(cat "$DOCTOR_LOG")" = 'doctor-fixable - +doctor-fixable --fix +doctor-fixable -' ] || fail "the repaired seed did not re-check after its repair"$'\n'"$(cat "$DOCTOR_LOG")" +pass "remote seeding proceeds once the repair closes every gap" # Provision and register the remote route from the captain-facing primary. out=$(FM_SECONDMATE_CHARTER='Own iOS delivery on the build Mac.' \ @@ -360,30 +465,76 @@ pass "mixed local and remote routes validate without migration" # host placement separately from that backend and arms the reply source. printf 'pi\n' > "$PARENT/config/crew-harness" launches_before_inherit=0 -[ ! -f "$TMUX_LOG" ] || launches_before_inherit=$(grep -c '^new-window' "$TMUX_LOG" || true) +[ ! -f "$HERDR_LOG" ] || launches_before_inherit=$(grep -c '^tab create' "$HERDR_LOG" || true) if FM_FAKE_SSH_MODE=inherit-partial remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate \ > "$TMP_ROOT/spawn-inherit-partial.out" 2>&1; then fail "remote spawn launched after ambiguous partial inheritance" fi launches_after_inherit=0 -[ ! -f "$TMUX_LOG" ] || launches_after_inherit=$(grep -c '^new-window' "$TMUX_LOG" || true) +[ ! -f "$HERDR_LOG" ] || launches_after_inherit=$(grep -c '^tab create' "$HERDR_LOG" || true) [ "$launches_before_inherit" -eq "$launches_after_inherit" ] \ || fail "remote spawn reached launch after ambiguous partial inheritance" assert_absent "$PARENT/state/ios.meta" "failed remote inheritance published launch metadata" out=$(remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate) -assert_contains "$out" 'remote=remote-mac backend=tmux' "remote spawn did not report separate host and backend dimensions" +assert_contains "$out" 'remote=remote-mac backend=herdr' "remote spawn did not report separate host and backend dimensions" assert_grep 'remote_host=remote-mac' "$PARENT/state/ios.meta" "parent metadata omitted the remote host" -assert_grep 'remote_backend=tmux' "$PARENT/state/ios.meta" "parent metadata omitted the remote-local backend" +assert_grep 'remote_backend=herdr' "$PARENT/state/ios.meta" "parent metadata omitted the remote-local backend" assert_grep 'window=remote:ios' "$PARENT/state/ios.meta" "parent metadata pretended the endpoint was local" assert_present "$PARENT/state/procevent/remote-reply-ios.source" "remote spawn did not arm its reply source" publish_healthy_watcher_identity "$PARENT/state" "$PARENT" "$ROOT/bin/fm-watch.sh" [ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh state ios)" = alive ] \ || fail "remote endpoint was not projected alive from its own host" -[ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh observe ios)" = fallback-idle ] \ +# Herdr reports a native agent state, so the delivery observation resolves +# without the rendered-output fallback a tmux endpoint needs. +[ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh observe ios)" = idle ] \ || fail "remote endpoint delivery observation did not execute on its own host" pass "remote spawn launches on the remote-local backend and records a host-qualified route" -rm -f "$TMP_ROOT/inherit.entered" "$TMP_ROOT/inherit.release" "$TMP_ROOT/inherit.payload" "$TMUX_STATE" +cp "$PARENT/state/ios.meta" "$TMP_ROOT/parent-ios-before-nonherdr.meta" +cp "$PARENT/data/secondmates.md" "$TMP_ROOT/registry-before-nonherdr.md" +set +e +FM_FAKE_SSH_MODE=launch-nonherdr-route remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate \ + > "$TMP_ROOT/spawn-nonherdr-route.out" 2>&1 +nonherdr_parent_rc=$? +set -e +[ "$nonherdr_parent_rc" -ne 0 ] || fail "parent accepted a non-herdr remote launch route" +assert_grep "remote launch returned backend 'tmux', expected herdr" "$TMP_ROOT/spawn-nonherdr-route.out" \ + "parent refusal did not name the returned remote backend" +cmp -s "$TMP_ROOT/parent-ios-before-nonherdr.meta" "$PARENT/state/ios.meta" \ + || fail "parent rewrote its endpoint metadata after a non-herdr route refusal" +cmp -s "$TMP_ROOT/registry-before-nonherdr.md" "$PARENT/data/secondmates.md" \ + || fail "parent removed or changed the registry route after a non-herdr route refusal" + +remote_route_meta="$REMOTE_HOME/state/parent-route/ios.meta" +cp "$remote_route_meta" "$TMP_ROOT/remote-ios-before-legacy.meta" +cat > "$remote_route_meta" < "$TMUX_STATE" +set +e +remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh launch ios codex - - herdr \ + > "$TMP_ROOT/legacy-alive-refusal.out" 2>&1 +legacy_alive_rc=$? +set -e +[ "$legacy_alive_rc" -ne 0 ] || fail "remote control reused an alive legacy tmux endpoint" +assert_grep "alive endpoint recorded on backend 'tmux'" "$TMP_ROOT/legacy-alive-refusal.out" \ + "remote refusal did not name the alive endpoint's recorded backend" +cmp -s "$TMP_ROOT/remote-ios-legacy-before-refusal.meta" "$remote_route_meta" \ + || fail "remote refusal changed the legacy endpoint metadata" +assert_present "$TMUX_STATE" "remote refusal killed the alive legacy endpoint" +cmp -s "$TMP_ROOT/registry-before-nonherdr.md" "$PARENT/data/secondmates.md" \ + || fail "remote legacy refusal removed or changed the registry route" +mv -f "$TMP_ROOT/remote-ios-before-legacy.meta" "$remote_route_meta" +rm -f "$TMUX_STATE" +pass "non-herdr remote endpoints are refused without changing either route" + +rm -f "$TMP_ROOT/inherit.entered" "$TMP_ROOT/inherit.release" "$TMP_ROOT/inherit.payload" cat > "$PARENT/data/captain-shared.md" <<'EOF' # Shared captain preferences This file is main-authoritative and maintained by the main firstmate. @@ -415,7 +566,7 @@ kill -0 "$spawn_config_push" 2>/dev/null \ || fail "config push bypassed the active remote spawn inheritance transaction" touch "$TMP_ROOT/inherit.release" wait "$spawn_concurrent" || fail "serialized remote spawn failed" -wait "$spawn_config_push" || fail "config push failed after serialized remote spawn" +wait "$spawn_config_push" || fail "config push failed after serialized remote spawn"$'\n'"$(cat "$TMP_ROOT/spawn-concurrent-push.out")" [ "$(tail -1 "$REMOTE_HOME/data/captain-shared.md")" = 'current post-spawn preference' ] \ || fail "stale spawn inheritance overwrote later config convergence" pass "remote spawn serializes inheritance through launch publication" @@ -432,7 +583,7 @@ set -e assert_grep 'do not resend' "$TMP_ROOT/send.err" "ambiguous remote send did not require same-host reconciliation" ssh_after_send=$(cat "$SSH_COUNT") [ "$ssh_after_send" -eq $((ssh_before_send + 1)) ] || fail "ambiguous remote send was retried" -CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$TMUX_LOG" | tail -1 | cut -d= -f2-) +CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$HERDR_LOG" | tail -1 | cut -d= -f2-) [ -n "$CORR" ] || fail "remote send did not carry a correlation token" phase=$(grep '^phase=' "$PARENT/state/pending-replies/$CORR" | cut -d= -f2-) [ "$phase" = delivery_unknown ] || fail "ambiguous remote send did not preserve its pending expectation" @@ -468,7 +619,7 @@ remote_env "$ROOT/bin/fm-bootstrap.sh" > "$TMP_ROOT/config-partial-retry.out" \ [ "$(cat "$REMOTE_HOME/config/crew-harness")" = grok ] \ || fail "bootstrap did not apply the remaining inherited file" assert_absent "$NUDGE_MARKER" "bootstrap cleared no remote reread marker after convergence" -PARTIAL_CONFIG_CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$TMUX_LOG" | tail -1 | cut -d= -f2-) +PARTIAL_CONFIG_CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$HERDR_LOG" | tail -1 | cut -d= -f2-) [ -n "$PARTIAL_CONFIG_CORR" ] || fail "bootstrap config reread did not carry a correlation token" printf 'done [corr=%s]: converged inherited config re-read\n' "$PARTIAL_CONFIG_CORR" >> "$REMOTE_HOME/state/parent-replies.status" remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null \ @@ -516,7 +667,7 @@ wait "$config_second" || fail "bootstrap inheritance transaction failed after wa pass "config push and bootstrap serialize remote inheritance convergence" printf 'codex\n' > "$PARENT/config/crew-harness" -touch "$TMP_ROOT/tmux-send-fail" +touch "$TMP_ROOT/herdr-send-fail" if remote_env "$ROOT/bin/fm-config-push.sh" > "$TMP_ROOT/config-push-fail.out" 2>&1; then fail "remote config push claimed success after its reread send failed" fi @@ -525,12 +676,12 @@ if [ ! -f "$NUDGE_MARKER" ]; then fail "failed remote config reread did not retain a retry marker" fi assert_grep 'remote=1' "$NUDGE_MARKER" "remote config reread marker lost its placement" -rm -f "$TMP_ROOT/tmux-send-fail" +rm -f "$TMP_ROOT/herdr-send-fail" remote_env "$ROOT/bin/fm-config-push.sh" > "$TMP_ROOT/config-push-retry.out" \ || fail "unchanged remote config push did not retry its pending reread" assert_absent "$NUDGE_MARKER" "successful remote config reread left its retry marker" assert_grep 'config-reread: sent' "$TMP_ROOT/config-push-retry.out" "remote config reread retry was not reported" -CONFIG_CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$TMUX_LOG" | tail -1 | cut -d= -f2-) +CONFIG_CORR=$(grep -Eo 'corr=[a-f0-9]{16}' "$HERDR_LOG" | tail -1 | cut -d= -f2-) [ -n "$CONFIG_CORR" ] || fail "remote config reread did not carry a correlation token" printf 'done [corr=%s]: inherited config re-read\n' "$CONFIG_CORR" >> "$REMOTE_HOME/state/parent-replies.status" remote_env "$ROOT/bin/fm-procevent.sh" start "$SID" >/dev/null \ @@ -591,8 +742,60 @@ assert_contains "$UPDATE_OUT" 'synced:' "remote update did not report a host-loc assert_present "$REMOTE_HOME/REMOTE_UPDATE_PROBE" "remote update did not materialize the code-root commit" pass "remote update imports and fast-forwards the persistent home on its configured host" +rm -f "$TMP_ROOT/doctor.repaired" +: > "$DOCTOR_LOG" +[ "$(FM_FAKE_SSH_MODE=doctor-fixable remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh state ios)" = unreadable ] \ + || fail "the stopped-server fixture did not make the pre-repair endpoint probe unreadable" +launches_before_repair=$(grep -c '^tab create' "$HERDR_LOG" || true) +BOOT_REPAIRED=$(FM_FAKE_SSH_MODE=doctor-fixable remote_env "$ROOT/bin/fm-bootstrap.sh") +[ "$(cat "$DOCTOR_LOG")" = 'doctor-fixable - +doctor-fixable --fix +doctor-fixable -' ] || fail "liveness did not check, repair, and re-check readiness before probing"$'\n'"$(cat "$DOCTOR_LOG")" +assert_not_contains "$BOOT_REPAIRED" 'SECONDMATE_LIVENESS: secondmate ios:' \ + "successful pre-probe readiness repair produced a liveness failure" +launches_after_repair=$(grep -c '^tab create' "$HERDR_LOG" || true) +[ "$launches_before_repair" -eq "$launches_after_repair" ] \ + || fail "readiness repair introduced a new remote relaunch point" +[ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh state ios)" = alive ] \ + || fail "the endpoint was not probed successfully after readiness repair" +pass "startup repairs remote readiness before probing without relaunching" + +remote_route_meta="$REMOTE_HOME/state/parent-route/ios.meta" +cp "$remote_route_meta" "$TMP_ROOT/remote-ios-before-liveness-legacy.meta" +cp "$PARENT/state/ios.meta" "$TMP_ROOT/parent-ios-before-liveness-legacy.meta" +cp "$PARENT/data/secondmates.md" "$TMP_ROOT/registry-before-liveness-legacy.md" +cat > "$remote_route_meta" < "$TMUX_STATE" +tmux_state_before=$(cat "$TMUX_STATE") +launches_before_legacy=$(grep -c '^tab create' "$HERDR_LOG" || true) +BOOT_LEGACY=$(remote_env "$ROOT/bin/fm-bootstrap.sh") +assert_contains "$BOOT_LEGACY" "SECONDMATE_LIVENESS: secondmate ios: skipped: alive remote endpoint is recorded on backend 'tmux'; migrate or retire it explicitly" \ + "liveness accepted an alive legacy remote backend" +cmp -s "$TMP_ROOT/remote-ios-liveness-legacy.meta" "$remote_route_meta" \ + || fail "liveness rewrote the alive legacy endpoint metadata" +cmp -s "$TMP_ROOT/parent-ios-before-liveness-legacy.meta" "$PARENT/state/ios.meta" \ + || fail "liveness rewrote the parent route metadata for an alive legacy endpoint" +cmp -s "$TMP_ROOT/registry-before-liveness-legacy.md" "$PARENT/data/secondmates.md" \ + || fail "liveness changed the registry route for an alive legacy endpoint" +[ "$(cat "$TMUX_STATE")" = "$tmux_state_before" ] \ + || fail "liveness changed or killed the alive legacy endpoint" +launches_after_legacy=$(grep -c '^tab create' "$HERDR_LOG" || true) +[ "$launches_before_legacy" -eq "$launches_after_legacy" ] \ + || fail "liveness relaunched an alive legacy endpoint" +mv -f "$TMP_ROOT/remote-ios-before-liveness-legacy.meta" "$remote_route_meta" +rm -f "$TMUX_STATE" +pass "startup reports alive legacy backends without changing their routes" + # Host loss maps to unknown/unavailable and never creates a local replacement. -launches_before=$(grep -c '^new-window' "$TMUX_LOG" || true) +launches_before=$(grep -c '^tab create' "$HERDR_LOG" || true) rm -rf -- "$PARENT/state/.watch.lock" rm -f -- "$PARENT/state/.last-watcher-beat" BOOT_UNAVAILABLE=$(FM_FAKE_SSH_MODE=unreachable remote_env "$ROOT/bin/fm-bootstrap.sh") @@ -604,8 +807,10 @@ printf '%s' "$UNAVAILABLE" | jq -e '.secondmate_current.records | any(.id == "io printf '%s' "$UNAVAILABLE" | jq -e '.tasks[] | select(.id == "ios") | .paths.home.present == null' >/dev/null \ || fail "unreachable remote home presence was not projected unknown" rm -f "$PARENT/state/.wake-queue" -launches_after=$(grep -c '^new-window' "$TMUX_LOG" || true) +launches_after=$(grep -c '^tab create' "$HERDR_LOG" || true) [ "$launches_before" -eq "$launches_after" ] || fail "unreachable projection attempted a replacement launch" +assert_present "$PARENT/state/ios.meta" "unreachable readiness removed the parent route metadata" +assert_grep '- ios ' "$PARENT/data/secondmates.md" "unreachable readiness removed the registry route" pass "unreachable remote state remains unknown with no local respawn or failover" # Retirement delegates its safety check to the remote home. An in-flight child diff --git a/tests/fm-remote-secondmate-trace-context.test.sh b/tests/fm-remote-secondmate-trace-context.test.sh index 8e97165d626..571e61694ec 100755 --- a/tests/fm-remote-secondmate-trace-context.test.sh +++ b/tests/fm-remote-secondmate-trace-context.test.sh @@ -7,12 +7,14 @@ # spawn_remote_secondmate, which hands the launch to the remote host. These # assertions drive the real chain - parent fm-spawn -> fm-on -> the real remote # entrypoint -> fm-remote-secondmate-control -> the remote host's own fm-spawn - -# against a fake tmux, so the carrier the remote pane receives is observable. +# against a fake herdr CLI, so the carrier the remote pane receives is observable. # See docs/verification/trace-context.md for the maintained coverage inventory. set -u # shellcheck source=tests/lib.sh . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# shellcheck source=tests/remote-herdr-fixture.sh +. "$(dirname "${BASH_SOURCE[0]}")/remote-herdr-fixture.sh" # shellcheck source=/dev/null . "$ROOT/bin/fm-trace-context-lib.sh" @@ -25,6 +27,8 @@ REMOTE_ROOT="$TMP_ROOT/remote-root" REMOTE_HOME="$TMP_ROOT/remote-home" SECOND_HOME="$TMP_ROOT/remote-home-2" FAKEBIN=$(fm_fakebin "$TMP_ROOT/fake") +HERDR_LOG="$TMP_ROOT/remote-herdr.log" +HERDR_STATE="$TMP_ROOT/remote-herdr.state" TMUX_LOG="$TMP_ROOT/remote-tmux.log" TMUX_STATE="$TMP_ROOT/remote-tmux.state" CLAIMS="$TMP_ROOT/claims" @@ -39,9 +43,11 @@ trap 'FM_HOME="$PARENT" FM_PROCEVENT_CLAIM_ROOT="$CLAIMS" "$ROOT/bin/fm-proceven tar --exclude=.git --exclude=.no-mistakes --exclude=data --exclude=state --exclude=config -cf - . ) | (cd "$REMOTE_ROOT" && tar -xf -) -# Fake tmux on the remote host. Every invocation is logged verbatim, so the -# pre-launch `export TRACEPARENT=` line and the launch literal's -# FM_TRACE_CONTEXT prefix are both observable exactly as the pane received them. +# The remote host runs the Herdr fixture, whose every invocation is logged +# verbatim, so the pre-launch `export TRACEPARENT=` line and the launch +# literal's FM_TRACE_CONTEXT prefix are both observable exactly as the pane +# received them. The tmux fixture below only keeps the remote home's own +# non-second-mate tooling resolvable. cat > "$REMOTE_ROOT/bin/tmux" </dev/null | tr '\0' '\n' | head -1 | grep -q '^fm-remote-doctor.sh$'; then + printf 'ok: remote second-mate readiness confirmed on this host\n' + exit 0 +fi exec "$FM_FAKE_REMOTE_ENTRYPOINT" "$@" SH chmod +x "$FAKEBIN/fake-ssh" @@ -133,10 +148,10 @@ freeze_parent_session() { # What the remote pane actually received, read back from the remote tmux log. remote_injected_traceparent() { - sed -n 's/.*export TRACEPARENT=\([0-9a-f-]*\).*/\1/p' "$TMUX_LOG" | tail -1 + sed -n 's/.*export TRACEPARENT=\([0-9a-f-]*\).*/\1/p' "$HERDR_LOG" | tail -1 } remote_launch_snapshot() { - grep -o 'FM_TRACE_CONTEXT=[a-z]*' "$TMUX_LOG" | tail -1 | cut -d= -f2 + grep -o 'FM_TRACE_CONTEXT=[a-z]*' "$HERDR_LOG" | tail -1 | cut -d= -f2 } meta_traceparent() { sed -n 's/^traceparent=//p' "$1"; } @@ -148,27 +163,27 @@ FM_SECONDMATE_CHARTER='Own iOS delivery on the build Mac.' \ # --- disabled: the remote route must stay byte-identically untraced ---------- freeze_parent_session -: > "$TMUX_LOG" +: > "$HERDR_LOG" remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate >/dev/null 2>&1 \ || fail "default-off remote secondmate spawn failed" assert_present "$PARENT/state/ios.meta" "default-off remote spawn published no parent metadata" ! grep -q '^traceparent=' "$PARENT/state/ios.meta" \ || fail "default-off remote spawn must not record a traceparent= line" -! grep -q 'export TRACEPARENT=' "$TMUX_LOG" \ +! grep -q 'export TRACEPARENT=' "$HERDR_LOG" \ || fail "default-off remote spawn must not export a carrier into the remote pane" ! grep -q '^traceparent=' "$REMOTE_HOME/state/parent-route/ios.meta" \ || fail "default-off remote spawn must not record a carrier on the remote host" [ "$(remote_launch_snapshot)" = off ] \ || fail "default-off remote spawn must deliver FM_TRACE_CONTEXT=off (got '$(remote_launch_snapshot)')" assert_absent "$REMOTE_HOME/config/trace-context" "default-off remote spawn inherited an enablement flag" -grep -q 'export GOTMPDIR=' "$TMUX_LOG" || fail "the remote spawn should still run (GOTMPDIR is always exported)" +grep -q 'export GOTMPDIR=' "$HERDR_LOG" || fail "the remote spawn should still run (GOTMPDIR is always exported)" pass "disabled: a remote-routed second mate records and receives no carrier and stays enabled-off end to end" # --- enabled: one carrier is recorded by the parent and received remotely ---- : > "$PARENT/config/trace-context" freeze_parent_session -rm -f "$TMUX_STATE" # the previous endpoint is gone; this is an ordinary relaunch -: > "$TMUX_LOG" +reset_remote_herdr_fixture "$HERDR_STATE" # the previous endpoint is gone; this is an ordinary relaunch +: > "$HERDR_LOG" remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate >/dev/null 2>&1 \ || fail "enabled remote secondmate spawn failed" @@ -187,9 +202,9 @@ fm_trace_context_valid "$INJECTED_TP" \ || fail "an enabled remote spawn must deliver FM_TRACE_CONTEXT=on (got '$(remote_launch_snapshot)')" assert_present "$REMOTE_HOME/config/trace-context" \ "an enabled remote launch did not inherit the enablement flag into the remote home" -GOTMP_LINE=$(grep -n 'export GOTMPDIR=' "$TMUX_LOG" | tail -1 | cut -d: -f1) -TP_LINE=$(grep -n 'export TRACEPARENT=' "$TMUX_LOG" | tail -1 | cut -d: -f1) -LAUNCH_LINE=$(grep -n 'FM_TRACE_CONTEXT=' "$TMUX_LOG" | tail -1 | cut -d: -f1) +GOTMP_LINE=$(grep -n 'export GOTMPDIR=' "$HERDR_LOG" | tail -1 | cut -d: -f1) +TP_LINE=$(grep -n 'export TRACEPARENT=' "$HERDR_LOG" | tail -1 | cut -d: -f1) +LAUNCH_LINE=$(grep -n 'FM_TRACE_CONTEXT=' "$HERDR_LOG" | tail -1 | cut -d: -f1) [ -n "$GOTMP_LINE" ] && [ -n "$TP_LINE" ] && [ -n "$LAUNCH_LINE" ] \ || fail "remote pane log missing GOTMPDIR/TRACEPARENT/launch lines" [ "$TP_LINE" -gt "$GOTMP_LINE" ] \ @@ -199,8 +214,8 @@ LAUNCH_LINE=$(grep -n 'FM_TRACE_CONTEXT=' "$TMUX_LOG" | tail -1 | cut -d: -f1) pass "enabled: a remote-routed second mate receives one carrier in its pane, identical to the parent's recorded identity, before launch" # --- relaunch stability on the remote path ---------------------------------- -rm -f "$TMUX_STATE" -: > "$TMUX_LOG" +reset_remote_herdr_fixture "$HERDR_STATE" +: > "$HERDR_LOG" remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate >/dev/null 2>&1 \ || fail "enabled remote secondmate relaunch failed" RELAUNCH_TP=$(meta_traceparent "$PARENT/state/ios.meta") @@ -221,8 +236,8 @@ FM_SECONDMATE_CHARTER='Own the second build Mac.' \ TRACEPARENT="$AMBIENT" \ remote_env "$ROOT/bin/fm-remote-home-seed.sh" ios2 remote-mac "$REMOTE_ROOT" "$SECOND_HOME" --no-projects >/dev/null \ || fail "second remote seed failed" -rm -f "$TMUX_STATE" -: > "$TMUX_LOG" +reset_remote_herdr_fixture "$HERDR_STATE" +: > "$HERDR_LOG" TRACEPARENT="$AMBIENT" remote_env "$ROOT/bin/fm-spawn.sh" ios2 --secondmate >/dev/null 2>&1 \ || fail "second remote secondmate spawn failed" SECOND_TP=$(meta_traceparent "$PARENT/state/ios2.meta") diff --git a/tests/remote-herdr-fixture.sh b/tests/remote-herdr-fixture.sh new file mode 100644 index 00000000000..26de3975b46 --- /dev/null +++ b/tests/remote-herdr-fixture.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# tests/remote-herdr-fixture.sh - the stateful herdr CLI fixture the remote +# second-mate suites install on their fake remote host. +# +# A remote second mate always launches on the Herdr backend +# (docs/remote-secondmates.md), so a remote-route test needs a herdr CLI on the +# remote code root's own bin directory. This fixture models the workspace, tab, +# pane, and agent facts bin/backends/herdr.sh actually reads, backed by a JSON +# state file mutated with real jq, using the same verified herdr behaviors as +# tests/fm-backend-herdr.test.sh's stateful fake: workspace create seeds one +# default tab and returns its tab and root pane in the same response, closing a +# tab's only pane closes the tab, and agent get reports agent_not_found for a +# pane no agent has registered on. +# +# Beyond that it models the pane IO a real launch performs. A pane reports a +# registered agent once anything has been typed into it, and submitting starts +# one turn: the next agent read reports working and the pane settles back to +# idle, which is the native transition the adapter confirms a submit with. +# +# Usage: +# . "$(dirname "${BASH_SOURCE[0]}")/remote-herdr-fixture.sh" +# install_remote_herdr_fixture \ +# +# +# Every invocation is appended verbatim to , so a test reads back what +# the remote pane received. Creating makes every pane write +# fail, which is how a test simulates an endpoint that cannot be reached. + +install_remote_herdr_fixture() { # + local remote_root=$1 state=$2 log=$3 send_fail=$4 socket=$5 script="$1/bin/herdr" + mkdir -p "$remote_root/bin" + cat > "$script" <> "$script" <<'SH' +printf '%s\n' "$*" >> "$LOG" +jq_state() { jq "$@" "$STATE"; } +save() { tmp="$STATE.tmp.$$"; cat > "$tmp" && mv "$tmp" "$STATE"; } +ws=""; label=""; cwd="" +args=("$@") +for ((i=0; i<${#args[@]}; i++)); do + case "${args[$i]}" in + --workspace) ws=${args[$((i+1))]:-} ;; + --label) label=${args[$((i+1))]:-} ;; + --cwd) cwd=${args[$((i+1))]:-} ;; + esac +done +case "${1:-} ${2:-}" in + "status --json") + printf '{"client":{"version":"0.7.5","protocol":16},"server":{"running":true}}\n' ;; + "server "*|"server") : ;; + "workspace list") jq_state '{result:{workspaces:.workspaces}}' ;; + "workspace create") + n=$(jq_state -r '.next'); wsid="w$n"; dn=$((n + 1)) + jq_state --arg wsid "$wsid" --arg wlabel "$label" --arg cwd "$cwd" \ + --arg tabid "$wsid:t$dn" --arg paneid "$wsid:p$dn" \ + '.workspaces += [{workspace_id:$wsid, label:$wlabel, cwd:$cwd}] + | .tabs += [{tab_id:$tabid, label:"1", workspace_id:$wsid, pane_id:$paneid}] + | .next = (.next + 2)' | save + printf '{"result":{"workspace":{"workspace_id":"%s","label":"%s"},"tab":{"tab_id":"%s"},"root_pane":{"pane_id":"%s"}}}\n' \ + "$wsid" "$label" "$wsid:t$dn" "$wsid:p$dn" + ;; + "tab list") jq_state --arg w "$ws" '{result:{tabs:[.tabs[]|select(.workspace_id==$w)]}}' ;; + "tab create") + n=$(jq_state -r '.next'); tabid="$ws:t$n"; paneid="$ws:p$n" + jq_state --arg w "$ws" --arg wlabel "$label" --arg cwd "$cwd" --arg tabid "$tabid" --arg paneid "$paneid" \ + '.tabs += [{tab_id:$tabid, label:$wlabel, workspace_id:$w, pane_id:$paneid, cwd:$cwd}] + | .next = (.next + 1)' | save + printf '{"result":{"tab":{"tab_id":"%s"},"root_pane":{"pane_id":"%s"}}}\n' "$tabid" "$paneid" + ;; + "tab close") + jq_state --arg t "${3:-}" '.tabs |= [.[]|select(.tab_id != $t)]' | save ;; + "pane list") + jq_state --arg w "$ws" '{result:{panes:[.tabs[]|select(.workspace_id==$w)|{pane_id:.pane_id, tab_id:.tab_id}]}}' ;; + "pane get") + pane=${3:-} + if [ "$(jq_state -r --arg p "$pane" '[.tabs[]|select(.pane_id==$p)]|length')" = 0 ]; then + printf '{"error":{"code":"pane_not_found","message":"%s"}}\n' "$pane" + else + printf '{"result":{"pane":{"pane_id":"%s"}}}\n' "$pane" + fi + ;; + "pane close") + jq_state --arg p "${3:-}" \ + '.tabs |= [.[]|select(.pane_id != $p)] + | .typed |= with_entries(select(.key != $p)) + | .working |= with_entries(select(.key != $p))' | save ;; + "pane send-text") + [ ! -f "$SEND_FAIL" ] || exit 1 + jq_state --arg p "${3:-}" '.typed[$p] = true' | save ;; + "pane send-keys") + [ ! -f "$SEND_FAIL" ] || exit 1 + jq_state --arg p "${3:-}" '.typed[$p] = true | .working[$p] = true' | save ;; + "pane read") printf '\n' ;; + "pane process-info") printf '{"result":{"process":{"name":"codex"}}}\n' ;; + "agent get") + pane=${3:-} + if [ "$(jq_state -r --arg p "$pane" '.working[$p] // false')" = true ]; then + jq_state --arg p "$pane" '.working |= with_entries(select(.key != $p))' | save + printf '{"result":{"agent":{"agent_status":"working"}}}\n' + elif [ "$(jq_state -r --arg p "$pane" '.typed[$p] // false')" = true ]; then + printf '{"result":{"agent":{"agent_status":"idle"}}}\n' + else + printf '{"error":{"code":"agent_not_found","message":"%s"}}\n' "$pane" + fi + ;; + "session list"*) + printf '{"sessions":[{"name":"default","running":true,"socket_path":"%s"}]}\n' "$SOCKET" ;; +esac +exit 0 +SH + chmod +x "$script" + reset_remote_herdr_fixture "$state" +} + +# reset_remote_herdr_fixture : return the fake host to "no workspaces, +# tabs, or panes", which is what a test means by "the previous endpoint is gone". +reset_remote_herdr_fixture() { # + printf '{"next":1,"workspaces":[],"tabs":[],"typed":{},"working":{}}\n' > "$1" +} From c8edff36b8466ea0fe547d3abf4b8aa330489986 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 01:43:54 -0700 Subject: [PATCH 3/8] fix: isolate remote secondmates in shared Herdr session (#1659) * Pin remote secondmates to fm-remote * no-mistakes(review): Fail closed on legacy remote Herdr endpoints * no-mistakes(review): Isolate fm-remote launch agent from interactive default * no-mistakes(document): Document shared remote Herdr retirement safety --- AGENTS.md | 2 +- bin/fm-remote-doctor.sh | 16 +- bin/fm-remote-secondmate-control.sh | 138 +++++++++++------- bin/fm-send.sh | 5 + bin/fm-spawn.sh | 11 +- docs/remote-secondmates.md | 16 +- tests/fm-remote-doctor.test.sh | 50 ++++++- ...fm-remote-secondmate-lifecycle-e2e.test.sh | 86 ++++++++++- tests/fm-send-strict.test.sh | 28 ++++ tests/remote-herdr-fixture.sh | 2 +- 10 files changed, 274 insertions(+), 80 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 14a97ad7fba..9f900897783 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -91,7 +91,7 @@ state/ volatile runtime signals; gitignored .turn-ended touched by turn-end hooks .grok-turnend-token firstmate-owned grok hook registry token for the task; removed by teardown .kimi-turnend-token firstmate-owned Kimi hook registry token for the task; removed by teardown - .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) + .meta written by fm-spawn: window=, endpoint_task_id=, worktree=, project=, harness=, model=, effort=, kind=, mode=, yolo=, tasktmp=; an optional traceparent= only when trace context is enabled (docs/configuration.md "Trace context propagation"); kind=secondmate also records home= and projects=, plus remote_host=/remote_root=/remote_backend=/remote_herdr_session=/remote_target= for a remote route; a non-default runtime backend records further backend-specific fields (docs/configuration.md "Runtime backend"; bin/fm-backend.sh, section 8); fm-pr-check, including through fm-pr-merge, records one canonical pr= and the forge's pr_head= when available (GitHub pull requests and GitLab merge requests; docs/gitlab-merge-watch.md); fm-x-link appends x_request=, x_request_ts=, x_followups=, and optional x_platform=/x_reply_max_chars= for an X-mode-originated task (section 14) .herdr-presentation quarantinable attempt and restart-binding journal for Herdr's optional visual projection; never task or endpoint authority; see docs/herdr-backend.md "Optional presentation spaces" .check.sh authenticated slow poll; the watcher dispatches validated PR data and the byte-identified X shim through trusted repository scripts, runs registered custom checks from hash-validated private snapshots, and rejects every other state check without execution .check-trust private content binding created by fm-check-register.sh for an intentional custom check diff --git a/bin/fm-remote-doctor.sh b/bin/fm-remote-doctor.sh index fc2ee785a1d..4e76c755d72 100755 --- a/bin/fm-remote-doctor.sh +++ b/bin/fm-remote-doctor.sh @@ -9,10 +9,11 @@ # recomposing it, so the entrypoint stays the single owner of that ordering and # the two can never drift. # -# A remote second mate always runs on the Herdr backend, so readiness is more -# than tool resolution. herdr must resolve, its server must be reachable, and on -# macOS the Firstmate-owned launch agent dev.firstmate.herdr at -# ~/Library/LaunchAgents/dev.firstmate.herdr.plist must exist, carry +# A remote second mate always runs on the Herdr backend in the dedicated +# fm-remote session, so readiness is more than tool resolution. herdr must +# resolve, that server must be reachable, and on macOS the Firstmate-owned +# launch agent dev.firstmate.herdr.fm-remote at +# ~/Library/LaunchAgents/dev.firstmate.herdr.fm-remote.plist must exist, carry # LimitLoadToSessionType=Aqua, and be loaded into the console user's gui/ # domain, so the server belongs to the GUI login session and survives logout and # SSH disconnection. SSH cannot create an Aqua session, so a host with no GUI @@ -53,8 +54,11 @@ SCRIPT_DIR=${SCRIPT_SELF%/*} SCRIPT_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR" && pwd -P) REQUIRED_TOOLS=(git jq) OPTIONAL_TOOLS=(tmux treehouse no-mistakes tasks-axi claude codex opencode pi grok kimi) -LAUNCH_AGENT_LABEL=dev.firstmate.herdr -HERDR_SESSION_NAME=default +LAUNCH_AGENT_LABEL=dev.firstmate.herdr.fm-remote +# The dedicated remote-secondmate session. The user's interactive Herdr work +# remains in the separate default session, which this readiness check never +# requires or changes. +HERDR_SESSION_NAME=fm-remote LAUNCH_AGENT_DIR="${HOME:-}/Library/LaunchAgents" LAUNCH_AGENT_PLIST="$LAUNCH_AGENT_DIR/$LAUNCH_AGENT_LABEL.plist" LAUNCH_AGENT_LOG_DIR="${HOME:-}/Library/Logs" diff --git a/bin/fm-remote-secondmate-control.sh b/bin/fm-remote-secondmate-control.sh index 9ba809eb32e..cce92873ef4 100755 --- a/bin/fm-remote-secondmate-control.sh +++ b/bin/fm-remote-secondmate-control.sh @@ -13,14 +13,19 @@ # fm-remote-secondmate-control.sh update # fm-remote-secondmate-control.sh retire [--force] # -# Remote placement ends here, but the second-mate agent itself always runs on -# the Herdr backend, so launch refuses any other selection rather than reading -# this home's config/backend; fm-spawn/fm-send/fm-teardown keep owning the local -# endpoint mechanics, and the home's own workers keep its ordinary backend -# selection. bin/fm-remote-doctor.sh owns that host's readiness for Herdr, and -# docs/remote-secondmates.md owns why. A private parent-route state directory -# stores only the remote secondmate agent's endpoint record; the home's own +# Remote placement ends here, but the second-mate agent always runs on the +# Herdr backend in the dedicated fm-remote session, so launch refuses any other +# selection rather than reading this home's config/backend. The interactive +# default session remains for the user's work. +# fm-spawn/fm-send/fm-teardown keep owning the local endpoint mechanics. +# The home's own workers keep their ordinary backend selection. +# bin/fm-remote-doctor.sh owns that host's readiness for Herdr. +# docs/remote-secondmates.md owns why. +# A private parent-route state directory stores only the remote secondmate +# agent's endpoint record; the home's own # state/*.meta remains reserved for workers the secondmate supervises. +# Retirement closes only this secondmate's panes or workspace and never +# stops fm-remote or removes a sibling secondmate's workspace or panes. # # The optional launch traceparent is the per-task W3C trace-context carrier the # PARENT home resolved for this secondmate; this host only delivers it to the @@ -35,6 +40,7 @@ FM_ROOT="${FM_ROOT_OVERRIDE:-$(cd "$SCRIPT_DIR/.." && pwd)}" TARGET_HOME=${FM_HOME:?FM_HOME is required} CONTROL_STATE="$TARGET_HOME/state/parent-route" CONTROL_DATA="$TARGET_HOME/data/.parent-route" +REMOTE_HERDR_SESSION=fm-remote # shellcheck source=bin/fm-backend.sh . "$SCRIPT_DIR/fm-backend.sh" @@ -58,26 +64,59 @@ validate_home() { # [allow-absent] meta_path() { printf '%s/%s.meta\n' "$CONTROL_STATE" "$1"; } +remote_endpoint_load() { + local id=$1 herdr_session + REMOTE_ENDPOINT_ERROR= + REMOTE_ENDPOINT_META=$(meta_path "$id") + if ! fm_backend_validate_task_endpoint "$REMOTE_ENDPOINT_META" "$id" 2>/dev/null; then + REMOTE_ENDPOINT_ERROR="remote secondmate $id endpoint metadata is invalid; refusing access until it is explicitly migrated" + return 1 + fi + REMOTE_ENDPOINT_BACKEND=$FM_BACKEND_VALIDATED_BACKEND + REMOTE_ENDPOINT_TARGET=$FM_BACKEND_VALIDATED_TARGET + if [ "$REMOTE_ENDPOINT_BACKEND" != herdr ]; then + REMOTE_ENDPOINT_ERROR="remote secondmate $id endpoint is recorded on backend '$REMOTE_ENDPOINT_BACKEND', expected 'herdr'; refusing access until it is explicitly migrated" + return 1 + fi + herdr_session=$(fm_backend_meta_exact_value "$REMOTE_ENDPOINT_META" herdr_session 2>/dev/null || true) + if [ "$herdr_session" != "$REMOTE_HERDR_SESSION" ]; then + REMOTE_ENDPOINT_ERROR="remote secondmate $id endpoint is recorded in Herdr session '${herdr_session:-missing}', expected '$REMOTE_HERDR_SESSION'; refusing access until it is explicitly migrated" + return 1 + fi + case "$REMOTE_ENDPOINT_TARGET" in + "$REMOTE_HERDR_SESSION":?*) ;; + *) + REMOTE_ENDPOINT_ERROR="remote secondmate $id endpoint target '$REMOTE_ENDPOINT_TARGET' is outside Herdr session '$REMOTE_HERDR_SESSION'; refusing access until it is explicitly migrated" + return 1 + ;; + esac +} + +remote_endpoint_require() { + remote_endpoint_load "$1" || die "$REMOTE_ENDPOINT_ERROR" +} + state_value() { # ; prints recovery-grade state - local id=$1 meta backend target + local id=$1 meta meta=$(meta_path "$id") [ -f "$meta" ] && [ ! -L "$meta" ] || { printf 'missing\n'; return 0; } - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - [ -n "$target" ] || { printf 'unreadable\n'; return 0; } - fm_backend_agent_state "$backend" "$target" 2>/dev/null || printf 'unreadable\n' + if ! remote_endpoint_load "$id"; then + printf 'error: %s\n' "$REMOTE_ENDPOINT_ERROR" >&2 + printf 'unverified\n' + return 0 + fi + fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" 2>/dev/null || printf 'unreadable\n' } print_route() { # - local meta=$1 backend target harness traceparent - meta=$(meta_path "$meta") - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - harness=$(fm_meta_get "$meta" harness) - traceparent=$(fm_meta_get "$meta" traceparent) + local id=$1 harness traceparent + remote_endpoint_require "$id" + harness=$(fm_meta_get "$REMOTE_ENDPOINT_META" harness) + traceparent=$(fm_meta_get "$REMOTE_ENDPOINT_META" traceparent) printf 'schema=fm-remote-secondmate-control.v1\n' - printf 'backend=%s\n' "$backend" - printf 'target=%s\n' "$target" + printf 'backend=%s\n' "$REMOTE_ENDPOINT_BACKEND" + printf 'target=%s\n' "$REMOTE_ENDPOINT_TARGET" + printf 'herdr_session=%s\n' "$REMOTE_HERDR_SESSION" printf 'harness=%s\n' "$harness" [ -z "$traceparent" ] || printf 'traceparent=%s\n' "$traceparent" } @@ -95,7 +134,7 @@ cmd_route() { cmd_launch() { local id=$1 harness=$2 model=$3 effort=$4 selected_backend=$5 traceparent=${6:-} - local current meta out backend target + local current meta out herdr_session validate_id "$id" validate_home "$id" @@ -108,19 +147,16 @@ cmd_launch() { mkdir -p "$CONTROL_STATE" "$CONTROL_DATA" meta=$(meta_path "$id") if [ -f "$meta" ]; then - current=$(state_value "$id") + remote_endpoint_require "$id" + current=$(fm_backend_agent_state "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" 2>/dev/null || printf 'unreadable\n') case "$current" in alive) - backend=$(fm_backend_of_meta "$meta") - [ "$backend" = herdr ] \ - || die "remote secondmate $id has an alive endpoint recorded on backend '$backend'; refusing reuse until it is explicitly migrated or retired" print_route "$id" return 0 ;; dead) - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - fm_backend_kill "$backend" "$target" 2>/dev/null || die "could not remove the confirmed agent-less endpoint" + fm_backend_kill "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" 2>/dev/null \ + || die "could not remove the confirmed agent-less endpoint" ;; missing) ;; *) die "remote endpoint state is $current; refusing duplicate launch" ;; @@ -130,7 +166,7 @@ cmd_launch() { [ "$model" = - ] || ARGS+=(--model "$model") [ "$effort" = - ] || ARGS+=(--effort "$effort") [ -z "$traceparent" ] || ARGS+=(--traceparent "$traceparent") - if ! out=$(FM_HOME="$FM_ROOT" FM_ROOT_OVERRIDE="$FM_ROOT" \ + if ! out=$(HERDR_SESSION="$REMOTE_HERDR_SESSION" FM_HOME="$FM_ROOT" FM_ROOT_OVERRIDE="$FM_ROOT" \ FM_STATE_OVERRIDE="$CONTROL_STATE" FM_DATA_OVERRIDE="$CONTROL_DATA" \ FM_CONFIG_OVERRIDE="$TARGET_HOME/config" FM_SKIP_SECONDMATE_INHERIT=1 \ "$SCRIPT_DIR/fm-spawn.sh" "${ARGS[@]}" 2>&1); then @@ -138,57 +174,47 @@ cmd_launch() { die "remote host-local secondmate launch failed" fi [ -f "$meta" ] || die "remote launch returned without endpoint metadata" + herdr_session=$(fm_meta_get "$meta" herdr_session) + [ "$herdr_session" = "$REMOTE_HERDR_SESSION" ] \ + || die "remote launch recorded Herdr session '${herdr_session:-missing}', expected '$REMOTE_HERDR_SESSION'" print_route "$id" } cmd_send() { - local id=$1 message=$2 meta backend target + local id=$1 message=$2 validate_id "$id" validate_home "$id" - meta=$(meta_path "$id") - [ -f "$meta" ] || die "remote secondmate has no endpoint metadata" - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - [ -n "$target" ] || die "remote secondmate endpoint is unreadable" + remote_endpoint_require "$id" FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \ - "$SCRIPT_DIR/fm-send.sh" "$target" "$message" + "$SCRIPT_DIR/fm-send.sh" "$REMOTE_ENDPOINT_TARGET" "$message" } cmd_key() { - local id=$1 key=$2 meta target + local id=$1 key=$2 validate_id "$id" validate_home "$id" - meta=$(meta_path "$id") - [ -f "$meta" ] || die "remote secondmate has no endpoint metadata" - target=$(fm_backend_target_of_meta "$meta") + remote_endpoint_require "$id" FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \ - "$SCRIPT_DIR/fm-send.sh" "$target" --key "$key" + "$SCRIPT_DIR/fm-send.sh" "$REMOTE_ENDPOINT_TARGET" --key "$key" } cmd_capture() { - local id=$1 lines=${2:-20} meta backend target + local id=$1 lines=${2:-20} validate_id "$id" validate_home "$id" case "$lines" in ''|*[!0-9]*|0) die "capture line count must be positive" ;; esac [ "$lines" -le 100 ] || die "capture line count exceeds 100" - meta=$(meta_path "$id") - [ -f "$meta" ] || die "remote secondmate has no endpoint metadata" - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - fm_backend_capture "$backend" "$target" "$lines" "fm-$id" | head -c 65536 + remote_endpoint_require "$id" + fm_backend_capture "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "$lines" "fm-$id" | head -c 65536 } cmd_observe() { - local id=$1 meta backend target harness + local id=$1 harness validate_id "$id" validate_home "$id" - meta=$(meta_path "$id") - [ -f "$meta" ] || die "remote secondmate has no endpoint metadata" - backend=$(fm_backend_of_meta "$meta") - target=$(fm_backend_target_of_meta "$meta") - harness=$(fm_meta_get "$meta" harness) - [ -n "$target" ] || die "remote secondmate endpoint is unreadable" - fm_pending_reply_backend_observation "$backend" "$target" "fm-$id" "$harness" + remote_endpoint_require "$id" + harness=$(fm_meta_get "$REMOTE_ENDPOINT_META" harness) + fm_pending_reply_backend_observation "$REMOTE_ENDPOINT_BACKEND" "$REMOTE_ENDPOINT_TARGET" "fm-$id" "$harness" printf '\n' } @@ -244,7 +270,7 @@ cmd_retire() { return 0 fi [ -z "$force" ] || [ "$force" = --force ] || usage - [ -f "$(meta_path "$id")" ] || die "remote secondmate has no endpoint metadata to retire safely" + remote_endpoint_require "$id" FM_HOME="$TARGET_HOME" FM_ROOT_OVERRIDE="$FM_ROOT" FM_STATE_OVERRIDE="$TARGET_HOME/state" \ FM_CONFIG_OVERRIDE="$TARGET_HOME/config" "$SCRIPT_DIR/fm-guard.sh" || true if [ -n "$force" ]; then diff --git a/bin/fm-send.sh b/bin/fm-send.sh index ccce65ea82b..52dabb4b348 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -165,6 +165,11 @@ fm_send_resolve_target() { # fi case "$raw" in + fm-*:*) + # A named Herdr session may itself begin with "fm-". Keep that explicit + # session:pane target on the validated backend-target path below rather + # than mistaking it for an unresolved task selector. + ;; fm-*) RESOLUTION_TRIED="meta=$STATE/$raw.meta; legacy-meta=$STATE/${raw#fm-}.meta; backend=none" echo "error: no metadata for $raw in $STATE (tried $RESOLUTION_TRIED); pass a well-formed explicit backend target only when targeting outside this firstmate home" >&2 diff --git a/bin/fm-spawn.sh b/bin/fm-spawn.sh index 35a8df2b248..f8e682ba51e 100755 --- a/bin/fm-spawn.sh +++ b/bin/fm-spawn.sh @@ -335,7 +335,7 @@ fi spawn_remote_secondmate() { local id=$1 remote host root home harness positional model effort backend out rc meta tmp - local remote_backend remote_target remote_harness registry_lock remote_lock remote_generation + local remote_backend remote_target remote_harness remote_herdr_session registry_lock remote_lock remote_generation local remote_traceparent remote_recorded_traceparent local -a launch_args id=${POS[0]:-} @@ -513,6 +513,7 @@ spawn_remote_secondmate() { remote_backend=$(printf '%s\n' "$out" | sed -n 's/^backend=//p' | tail -1) remote_target=$(printf '%s\n' "$out" | sed -n 's/^target=//p' | tail -1) remote_harness=$(printf '%s\n' "$out" | sed -n 's/^harness=//p' | tail -1) + remote_herdr_session=$(printf '%s\n' "$out" | sed -n 's/^herdr_session=//p' | tail -1) if [ "$remote_backend" != herdr ]; then fm_lock_release "$remote_lock" || true fm_lock_release "$registry_lock" || true @@ -527,6 +528,13 @@ spawn_remote_secondmate() { echo "error: remote launch returned malformed route metadata; preserving the remote route for reconciliation" >&2 return 1 } + if [ "$remote_herdr_session" != fm-remote ] || [ "${remote_target%%:*}" != "$remote_herdr_session" ]; then + fm_lock_release "$remote_lock" || true + fm_lock_release "$registry_lock" || true + fm_lock_release "$SPAWN_TASK_LOCK" || true + echo "error: remote launch returned Herdr session '${remote_herdr_session:-missing}', expected 'fm-remote'; preserving the remote route for reconciliation" >&2 + return 1 + fi # Record what the remote endpoint ACTUALLY carries, read back from its own # launch, rather than what this side hoped to deliver. That keeps the #995 # guarantee that the recorded carrier is the identity the child received even @@ -553,6 +561,7 @@ spawn_remote_secondmate() { echo "remote_host=$host" echo "remote_root=$root" echo "remote_backend=$remote_backend" + echo "remote_herdr_session=$remote_herdr_session" echo "remote_target=$remote_target" [ -z "$remote_recorded_traceparent" ] || echo "traceparent=$remote_recorded_traceparent" } > "$tmp" diff --git a/docs/remote-secondmates.md b/docs/remote-secondmates.md index fa31953aba8..f641569501b 100644 --- a/docs/remote-secondmates.md +++ b/docs/remote-secondmates.md @@ -4,9 +4,11 @@ Remote second mates place a whole persistent Firstmate home on another SSH-reach The primary still owns routing and supervision, while the remote home owns its own projects, backlog, and workers. Firstmate does not support placing an individual worker remotely or failing a remote route over to a local replacement. -The remote second-mate agent itself always runs on the [Herdr backend](herdr-backend.md), and every path that provisions or launches one refuses a host that is not ready for it. -Herdr's server belongs to the host's own GUI login session rather than to the SSH connection, so the agent's endpoint survives every disconnection the primary's supervision depends on. -Local second mates are unaffected and keep their ordinary backend selection, as do the workers a remote second mate supervises inside its own home. +The remote second-mate agent itself always runs on the [Herdr backend](herdr-backend.md) in the shared `fm-remote` session, and every path that provisions or launches one refuses a host that is not ready for it. +`fm-remote` is reserved for remote fleet work and must not be used for personal work. +The user's interactive Herdr session remains `default` and is not a remote-secondmate prerequisite. +Herdr's remote-session server belongs to the host's own GUI login session rather than to the SSH connection, so the agent's endpoint survives every disconnection the primary's supervision depends on. +Local second mates are unaffected and keep their ordinary backend and session selection, as do the workers a remote second mate supervises inside its own home. ## Prerequisites @@ -81,7 +83,8 @@ The script's own header owns the full line protocol. bin/fm-on.sh fm-remote-doctor.sh --fix ``` -It writes and reloads the Firstmate-owned launch agent `dev.firstmate.herdr` at `~/Library/LaunchAgents/dev.firstmate.herdr.plist`, scoped with `LimitLoadToSessionType=Aqua` so it belongs to the GUI login session, bootstraps and starts it in `gui/`, starts the herdr server directly on a host where no launch agent applies, and recreates the `~/.local/bin/fm-remote-entrypoint.sh` symlink when it is absent. +It writes and reloads the Firstmate-owned launch agent `dev.firstmate.herdr.fm-remote` at `~/Library/LaunchAgents/dev.firstmate.herdr.fm-remote.plist`, scoped with `LimitLoadToSessionType=Aqua` so it belongs to the GUI login session, bootstraps and starts the `fm-remote` server in `gui/`, starts that server directly on a host where no launch agent applies, and recreates the `~/.local/bin/fm-remote-entrypoint.sh` symlink when it is absent. +The dedicated launch agent owns only the remote-secondmate server and does not inspect, rewrite, start, stop, or require the user's interactive `default` session or its `dev.firstmate.herdr` launch agent. It re-derives every check from the host afterwards, so what it prints is the state after the repair rather than the intent of one. These steps are never automated and are always reported rather than silently attempted, because SSH cannot create a GUI session from nothing: @@ -122,8 +125,10 @@ Launch or recover the remote second mate with the same command used for a local bin/fm-spawn.sh --secondmate ``` -The primary resolves the verified secondmate harness and optional model and effort, runs the same readiness gate the seed runs, transfers the inherited-material allowlist, and asks the remote host to launch on Herdr. +The primary resolves the verified secondmate harness and optional model and effort, runs the same readiness gate the seed runs, transfers the inherited-material allowlist, and asks the remote host to launch on Herdr in `fm-remote`. +All remote secondmates on one host share `fm-remote` and retain separate `2ndmate-` workspaces inside it. An explicit request for any other backend is refused rather than honored, and the remote host refuses one too. +An existing remote endpoint recorded in another Herdr session, including `default`, is classified as unverified and left untouched; launch, liveness recovery, control, and retirement refuse it until an operator explicitly migrates it instead of attempting a live cutover. A launch after a host has drifted out of readiness fails with the doctor's own gap text instead of leaving a half-created endpoint. Raw launch commands are not accepted for remote secondmates. Backends that already refuse secondmate launch, currently Orca and cmux, remain unsupported on the remote host. @@ -179,6 +184,7 @@ bin/fm-teardown.sh ``` Retirement is executed on the configured host and refuses while the remote home has child work, while the primary has an unfinished backlog outbox, or while a routed reply remains unresolved. +It closes only the retiring secondmate's panes or `2ndmate-` workspace in `fm-remote`; it never stops the shared session or removes a sibling secondmate's workspace or panes. SSH exit 255 preserves both the route and local records because completion is unknown. `--force` remains the explicit discard path and requires the same captain authority as local secondmate discard. No generic remote delete or write surface exists: remote writes are confined to inherited allowlist files and backlog handoff scratch files, and remote home removal is reachable only through guarded secondmate retirement. diff --git a/tests/fm-remote-doctor.test.sh b/tests/fm-remote-doctor.test.sh index 6247846cb14..c05eed4896c 100755 --- a/tests/fm-remote-doctor.test.sh +++ b/tests/fm-remote-doctor.test.sh @@ -13,7 +13,8 @@ set -u command -v jq >/dev/null 2>&1 || { echo "skip: jq not found (the herdr adapter parses its JSON)"; exit 0; } TMP_ROOT=$(fm_test_tmproot fm-remote-doctor) -LABEL=dev.firstmate.herdr +LABEL=dev.firstmate.herdr.fm-remote +INTERACTIVE_LABEL=dev.firstmate.herdr CASE_N=0 # A fixture must be able to present a host with NO herdr, so the doctor never @@ -40,6 +41,7 @@ new_case() { CASE_FORBIDDEN_LOG="$CASE_STATE/forbidden.log" CASE_HERDR_RUNNING="$CASE_STATE/herdr.running" CASE_PLIST="$CASE_HOME/Library/LaunchAgents/$LABEL.plist" + CASE_INTERACTIVE_PLIST="$CASE_HOME/Library/LaunchAgents/$INTERACTIVE_LABEL.plist" mkdir -p "$CASE_BIN" "$CASE_HOME" "$CASE_STATE" printf 'false\n' > "$CASE_HERDR_RUNNING" : > "$CASE_LAUNCHCTL_LOG" @@ -59,14 +61,24 @@ domain=${2:-} case "${1:-}" in print) case "$domain" in - */*/*) [ -f "$FM_FAKE_STATE/loaded-contract" ] || exit 113; cat "$FM_FAKE_STATE/loaded-contract" ;; + */dev.firstmate.herdr.fm-remote) + [ -f "$FM_FAKE_STATE/loaded-contract" ] || exit 113 + cat "$FM_FAKE_STATE/loaded-contract" + ;; + */dev.firstmate.herdr) + [ -f "$FM_FAKE_STATE/interactive-loaded" ] || exit 113 + printf 'interactive default job\n' + ;; *) [ -f "$FM_FAKE_STATE/gui-session" ] || exit 113 ;; esac exit 0 ;; bootout) [ ! -f "$FM_FAKE_STATE/bootout-fail" ] || { printf 'Boot-out failed: operation not permitted\n' >&2; exit 6; } - rm -f "$FM_FAKE_STATE/loaded-contract" + case "$domain" in + */dev.firstmate.herdr.fm-remote) rm -f "$FM_FAKE_STATE/loaded-contract" ;; + */dev.firstmate.herdr) rm -f "$FM_FAKE_STATE/interactive-loaded" ;; + esac exit 0 ;; bootstrap) @@ -80,7 +92,7 @@ arguments = { $FM_FAKE_HERDR_BIN server --session - default + fm-remote } stdout path = $FM_FAKE_LAUNCH_AGENT_LOG stderr path = $FM_FAKE_LAUNCH_AGENT_LOG @@ -179,7 +191,7 @@ arguments = { $herdr_bin server --session - default + fm-remote } stdout path = $CASE_HOME/Library/Logs/$LABEL.log stderr path = $CASE_HOME/Library/Logs/$LABEL.log @@ -212,6 +224,25 @@ pass "a missing herdr CLI is a human gap that --fix never claims to close" # --- an absent launch agent is a fixable gap that --fix installs ------------- new_case Darwin with-herdr gui +mkdir -p "$(dirname "$CASE_INTERACTIVE_PLIST")" +cat > "$CASE_INTERACTIVE_PLIST" < + + + Label + $INTERACTIVE_LABEL + ProgramArguments + + $CASE_BIN/herdr + server + --session + default + + + +XML +cp "$CASE_INTERACTIVE_PLIST" "$CASE_STATE/interactive-before.plist" +touch "$CASE_STATE/interactive-loaded" doctor expect_code 1 "$DOCTOR_RC" "a host with no launch agent was reported ready" assert_contains "$DOCTOR_OUT" 'check herdr=ok:' "the fake herdr CLI was not detected" @@ -236,9 +267,16 @@ assert_present "$CASE_PLIST" "--fix reported success without writing the plist" assert_grep 'Aqua' "$CASE_PLIST" "the written plist is not Aqua-scoped" assert_grep "$LABEL" "$CASE_PLIST" "the written plist does not carry the Firstmate label" assert_grep 'server' "$CASE_PLIST" "the written plist does not run a herdr server" +assert_grep 'fm-remote' "$CASE_PLIST" "the written plist does not pin the remote-secondmate session" +assert_no_grep 'default' "$CASE_PLIST" "the written plist pins the interactive default session" assert_grep "gui/$(id -u)" "$CASE_LAUNCHCTL_LOG" "the launch agent was not bootstrapped into the GUI domain" +cmp -s "$CASE_STATE/interactive-before.plist" "$CASE_INTERACTIVE_PLIST" \ + || fail "the fm-remote repair rewrote the interactive default launch agent" +assert_present "$CASE_STATE/interactive-loaded" "the fm-remote repair unloaded the interactive default launch agent" +assert_no_grep "gui/$(id -u)/$INTERACTIVE_LABEL$" "$CASE_LAUNCHCTL_LOG" \ + "the fm-remote repair inspected or controlled the interactive default launch agent" assert_no_dangerous_calls "the repair reached for auto-login, FileVault, or the keychain" -pass "--fix installs, Aqua-scopes, loads, and starts the Firstmate herdr launch agent" +pass "--fix installs the dedicated fm-remote launch agent without touching default" PLIST_BEFORE=$(cat "$CASE_PLIST") : > "$CASE_LAUNCHCTL_LOG" diff --git a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh index 15e8d4e3c09..4f30e749298 100755 --- a/tests/fm-remote-secondmate-lifecycle-e2e.test.sh +++ b/tests/fm-remote-secondmate-lifecycle-e2e.test.sh @@ -198,6 +198,15 @@ case "${FM_FAKE_SSH_MODE:-normal}:$command_name:$command_rel" in printf 'harness=codex\n' exit 0 ;; + launch-default-session-route:fm-remote-secondmate-control.sh:*) + [ "$_command_action" = launch ] || exit 93 + printf 'schema=fm-remote-secondmate-control.v1\n' + printf 'backend=herdr\n' + printf 'target=default:w1:p2\n' + printf 'herdr_session=default\n' + printf 'harness=codex\n' + exit 0 + ;; provision-block-fail:fm-remote-home-provision.sh:*) touch "$FM_FAKE_SEED_ENTERED" while [ ! -f "$FM_FAKE_SEED_RELEASE" ]; do sleep 0.02; done @@ -479,6 +488,11 @@ out=$(remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate) assert_contains "$out" 'remote=remote-mac backend=herdr' "remote spawn did not report separate host and backend dimensions" assert_grep 'remote_host=remote-mac' "$PARENT/state/ios.meta" "parent metadata omitted the remote host" assert_grep 'remote_backend=herdr' "$PARENT/state/ios.meta" "parent metadata omitted the remote-local backend" +assert_grep 'remote_herdr_session=fm-remote' "$PARENT/state/ios.meta" "parent metadata omitted the pinned remote Herdr session" +assert_grep 'remote_target=fm-remote:' "$PARENT/state/ios.meta" "parent metadata did not record an fm-remote endpoint" +assert_grep 'herdr_session=fm-remote' "$REMOTE_HOME/state/parent-route/ios.meta" "remote metadata did not record the pinned Herdr session" +assert_grep '--session fm-remote' "$HERDR_LOG" "remote launch did not target the fm-remote session" +assert_no_grep '--session default' "$HERDR_LOG" "remote launch targeted the interactive default session" assert_grep 'window=remote:ios' "$PARENT/state/ios.meta" "parent metadata pretended the endpoint was local" assert_present "$PARENT/state/procevent/remote-reply-ios.source" "remote spawn did not arm its reply source" publish_healthy_watcher_identity "$PARENT/state" "$PARENT" "$ROOT/bin/fm-watch.sh" @@ -490,6 +504,42 @@ publish_healthy_watcher_identity "$PARENT/state" "$PARENT" "$ROOT/bin/fm-watch.s || fail "remote endpoint delivery observation did not execute on its own host" pass "remote spawn launches on the remote-local backend and records a host-qualified route" +remote_route_meta="$REMOTE_HOME/state/parent-route/ios.meta" +cp "$remote_route_meta" "$TMP_ROOT/remote-ios-before-default-session.meta" +legacy_pane=$(sed -n 's/^herdr_pane_id=//p' "$remote_route_meta") +awk -v pane="$legacy_pane" ' + /^window=/ { print "window=default:" pane; next } + /^herdr_session=/ { print "herdr_session=default"; next } + { print } +' "$TMP_ROOT/remote-ios-before-default-session.meta" > "$remote_route_meta" +cp "$HERDR_LOG" "$TMP_ROOT/herdr-before-default-session.log" +[ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh state ios 2>/dev/null)" = unverified ] \ + || fail "legacy default-session metadata was not classified unverified" +if remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh route ios >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh send ios probe >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh key ios Enter >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh capture ios >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh observe ios >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh retire ios --force >/dev/null 2>&1 \ + || remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh launch ios codex - - herdr >/dev/null 2>&1; then + fail "legacy default-session metadata remained operational" +fi +cmp -s "$TMP_ROOT/herdr-before-default-session.log" "$HERDR_LOG" \ + || fail "legacy default-session metadata caused a Herdr operation" +assert_present "$REMOTE_HOME" "refused legacy retirement removed the remote home" +assert_grep 'herdr_session=default' "$remote_route_meta" "refused legacy retirement rewrote endpoint metadata" + +awk -v pane="$legacy_pane" ' + /^window=/ { print "window=default:" pane; next } + { print } +' "$TMP_ROOT/remote-ios-before-default-session.meta" > "$remote_route_meta" +[ "$(remote_env "$ROOT/bin/fm-on.sh" ios fm-remote-secondmate-control.sh state ios 2>/dev/null)" = unverified ] \ + || fail "mismatched fm-remote target was not classified unverified" +cmp -s "$TMP_ROOT/herdr-before-default-session.log" "$HERDR_LOG" \ + || fail "mismatched fm-remote target caused a Herdr operation" +mv -f "$TMP_ROOT/remote-ios-before-default-session.meta" "$remote_route_meta" +pass "legacy and mismatched remote endpoints fail closed before backend access" + cp "$PARENT/state/ios.meta" "$TMP_ROOT/parent-ios-before-nonherdr.meta" cp "$PARENT/data/secondmates.md" "$TMP_ROOT/registry-before-nonherdr.md" set +e @@ -505,6 +555,17 @@ cmp -s "$TMP_ROOT/parent-ios-before-nonherdr.meta" "$PARENT/state/ios.meta" \ cmp -s "$TMP_ROOT/registry-before-nonherdr.md" "$PARENT/data/secondmates.md" \ || fail "parent removed or changed the registry route after a non-herdr route refusal" +set +e +FM_FAKE_SSH_MODE=launch-default-session-route remote_env "$ROOT/bin/fm-spawn.sh" ios --secondmate \ + > "$TMP_ROOT/spawn-default-session-route.out" 2>&1 +default_session_parent_rc=$? +set -e +[ "$default_session_parent_rc" -ne 0 ] || fail "parent accepted an interactive default-session remote route" +assert_grep "remote launch returned Herdr session 'default', expected 'fm-remote'" "$TMP_ROOT/spawn-default-session-route.out" \ + "parent refusal did not name the default session" +cmp -s "$TMP_ROOT/parent-ios-before-nonherdr.meta" "$PARENT/state/ios.meta" \ + || fail "parent rewrote its endpoint metadata after a default-session route refusal" + remote_route_meta="$REMOTE_HOME/state/parent-route/ios.meta" cp "$remote_route_meta" "$TMP_ROOT/remote-ios-before-legacy.meta" cat > "$remote_route_meta" < "$TMUX_STATE" tmux_state_before=$(cat "$TMUX_STATE") launches_before_legacy=$(grep -c '^tab create' "$HERDR_LOG" || true) BOOT_LEGACY=$(remote_env "$ROOT/bin/fm-bootstrap.sh") -assert_contains "$BOOT_LEGACY" "SECONDMATE_LIVENESS: secondmate ios: skipped: alive remote endpoint is recorded on backend 'tmux'; migrate or retire it explicitly" \ +assert_contains "$BOOT_LEGACY" "SECONDMATE_LIVENESS: secondmate ios: skipped: remote endpoint state is unverified on remote-mac" \ "liveness accepted an alive legacy remote backend" cmp -s "$TMP_ROOT/remote-ios-liveness-legacy.meta" "$remote_route_meta" \ || fail "liveness rewrote the alive legacy endpoint metadata" @@ -815,10 +876,20 @@ pass "unreachable remote state remains unknown with no local respawn or failover # Retirement delegates its safety check to the remote home. An in-flight child # record refuses cleanup and preserves both machines' durable routes. +# A sibling remote secondmate workspace shares fm-remote and must survive every +# refusal and the eventual successful retirement of ios. # This fixture overrides FM_ROOT for transport, so teardown's root-owned guard # sees the fixture root rather than the source script path used by fm-send. publish_healthy_watcher_identity "$PARENT/state" "$PARENT" "$REMOTE_ROOT/bin/fm-watch.sh" resolve_ios_pending +SIBLING_CREATE=$("$REMOTE_ROOT/bin/herdr" workspace create --cwd "$REMOTE_ROOT" \ + --label 2ndmate-macos --no-focus --session fm-remote) +SIBLING_WORKSPACE=$(printf '%s' "$SIBLING_CREATE" | jq -r '.result.workspace.workspace_id') +SIBLING_PANE=$(printf '%s' "$SIBLING_CREATE" | jq -r '.result.root_pane.pane_id') +[ -n "$SIBLING_WORKSPACE" ] && [ "$SIBLING_WORKSPACE" != null ] \ + || fail "the shared-session sibling fixture did not create a workspace" +[ -n "$SIBLING_PANE" ] && [ "$SIBLING_PANE" != null ] \ + || fail "the shared-session sibling fixture did not create a pane" printf 'kind=ship\n' > "$REMOTE_HOME/state/child.meta" rm -rf "$PARENT/state/procevent" : > "$PARENT/state/procevent" @@ -903,6 +974,13 @@ fi assert_absent "$REMOTE_HOME" "remote retirement did not remove the remote home" assert_absent "$PARENT/state/ios.meta" "remote retirement did not remove parent metadata" assert_no_grep '- ios ' "$PARENT/data/secondmates.md" "remote retirement did not remove the registry route" -pass "remote retirement refuses child work, then cleans the same host through existing guards" +jq -e --arg workspace "$SIBLING_WORKSPACE" --arg pane "$SIBLING_PANE" ' + any(.workspaces[]; .workspace_id == $workspace and .label == "2ndmate-macos") + and any(.tabs[]; .workspace_id == $workspace and .pane_id == $pane) +' "$HERDR_STATE" >/dev/null \ + || fail "remote retirement removed the sibling secondmate workspace or pane from fm-remote" +assert_no_grep 'session stop' "$HERDR_LOG" "remote retirement stopped the shared fm-remote session" +assert_no_grep 'server stop' "$HERDR_LOG" "remote retirement stopped the shared fm-remote server" +pass "remote retirement refuses child work, then removes only its own endpoint while a shared-session sibling survives" echo "ALL TESTS PASSED" diff --git a/tests/fm-send-strict.test.sh b/tests/fm-send-strict.test.sh index 1faf98a0ce2..d65569c6199 100755 --- a/tests/fm-send-strict.test.sh +++ b/tests/fm-send-strict.test.sh @@ -59,6 +59,17 @@ esac exit 0 SH chmod +x "$fb/tmux" + cat > "$fb/herdr" <<'SH' +#!/usr/bin/env bash +set -u +printf '%s\n' "$*" >> "$FM_HERDR_LOG" +case "${1:-} ${2:-}" in + "status --json") printf '{"client":{"version":"0.7.5","protocol":16},"server":{"running":true}}\n' ;; + "pane get") printf '{"result":{"pane":{"pane_id":"%s"}}}\n' "${3:-}" ;; + "pane send-keys") : ;; +esac +SH + chmod +x "$fb/herdr" cat > "$fb/sleep" <<'SH' #!/usr/bin/env bash exit 0 @@ -147,6 +158,22 @@ test_unmatched_single_colon_target_must_exist() { pass "fm-send strict: unmatched single-colon explicit targets must verify live before sending" } +test_fm_prefixed_herdr_session_is_an_explicit_target() { + local dir fb home err log herdr_log rc + dir="$TMP_ROOT/fm-remote-explicit"; mkdir -p "$dir" + fb=$(make_stubs "$dir"); home=$(setup_home fmremote); err="$dir/send.err"; log="$dir/tmux.log"; herdr_log="$dir/herdr.log" + : > "$log" + : > "$herdr_log" + + PATH="$fb:$PATH" FM_HOME="$home" FM_ROOT_OVERRIDE="$home" FM_TMUX_LOG="$log" FM_HERDR_LOG="$herdr_log" FM_SEND_SETTLE=0 \ + "$SEND" fm-remote:w1:p2 --key Enter >/dev/null 2>"$err"; rc=$? + expect_code 0 "$rc" "an fm-prefixed Herdr session target should be accepted as explicit" + assert_grep 'pane get w1:p2 --session fm-remote' "$herdr_log" "fm-prefixed Herdr target was not verified in its session" + assert_grep 'pane send-keys w1:p2 enter --session fm-remote' "$herdr_log" "fm-prefixed Herdr target was not sent its key in its session" + assert_no_grep '--session default' "$herdr_log" "fm-prefixed Herdr target fell back to the default session" + pass "fm-send strict: fm-prefixed Herdr sessions remain explicit backend targets" +} + test_healthy_fm_id_send_still_works() { local dir fb home err log rc got dir="$TMP_ROOT/healthy"; mkdir -p "$dir" @@ -168,4 +195,5 @@ test_unset_fm_home_fails test_unresolvable_target_does_not_tmux_fallback test_prefixless_herdr_pane_id_fails test_unmatched_single_colon_target_must_exist +test_fm_prefixed_herdr_session_is_an_explicit_target test_healthy_fm_id_send_still_works diff --git a/tests/remote-herdr-fixture.sh b/tests/remote-herdr-fixture.sh index 26de3975b46..b01419066cc 100644 --- a/tests/remote-herdr-fixture.sh +++ b/tests/remote-herdr-fixture.sh @@ -110,7 +110,7 @@ case "${1:-} ${2:-}" in fi ;; "session list"*) - printf '{"sessions":[{"name":"default","running":true,"socket_path":"%s"}]}\n' "$SOCKET" ;; + printf '{"sessions":[{"name":"default","running":true,"socket_path":"%s"},{"name":"fm-remote","running":true,"socket_path":"%s"}]}\n' "$SOCKET" "$SOCKET" ;; esac exit 0 SH From a83be60cfb3ac4cea560597ff2fe0e4c9b8cc0e6 Mon Sep 17 00:00:00 2001 From: Kun Chen <3233006+kunchenguid@users.noreply.github.com> Date: Tue, 4 Aug 2026 04:22:02 -0700 Subject: [PATCH 4/8] feat: route remote commands through an Aqua job worker (#1660) * feat: run remote commands through Aqua job worker * no-mistakes(review): Enforce remote job deadlines and safe worker shutdown * no-mistakes(review): Refresh stale workers and harden dependency-free supervision * no-mistakes(review): Harden worker ownership recovery and shutdown quarantine * no-mistakes(review): Fix doctor bootstrap, harness repair, and output draining * no-mistakes(review): Probe doctor tools through authenticated worker bootstrap * no-mistakes(review): Refresh stale workers before doctor tool probes * no-mistakes(review): Recover stopped quarantines and extend job deadlines * no-mistakes(review): Separate queue and execution timeout windows * no-mistakes(review): Supervise Linux worker crashes and bind root identity * no-mistakes(review): Resolve authorized Nix profile bin links * no-mistakes(review): Clarify Nix path resolution documentation * no-mistakes(review): Harden PATH safety and nvm selection * no-mistakes(review): Honor nvm system defaults and refresh doctor digest * no-mistakes(review): Keep workers ready during active jobs * no-mistakes(review): Bound pre-execution validation by job timeout * no-mistakes(document): Clarify remote worker documentation * no-mistakes(lint): Fix remote worker ShellCheck diagnostics * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes * no-mistakes: apply CI fixes --- bin/fm-backlog-handoff.sh | 2 +- bin/fm-bootstrap.sh | 6 +- bin/fm-fleet-snapshot.sh | 4 +- bin/fm-pending-reply-lib.sh | 2 +- bin/fm-procevent-remote-reply.sh | 4 +- bin/fm-remote-doctor.sh | 348 ++++++- bin/fm-remote-entrypoint.sh | 235 ++--- bin/fm-remote-inherit-push.sh | 3 +- bin/fm-remote-job-lib.sh | 922 ++++++++++++++++++ bin/fm-remote-job-worker.sh | 684 +++++++++++++ bin/fm-remote-readiness-lib.sh | 6 +- bin/fm-send.sh | 4 +- bin/fm-spawn.sh | 2 +- bin/fm-teardown.sh | 4 +- bin/fm-test-run.sh | 2 +- bin/fm-update.sh | 2 +- docs/architecture.md | 2 +- docs/remote-secondmates.md | 46 +- docs/scripts.md | 6 +- tests/fm-on.test.sh | 190 +++- tests/fm-remote-backlog-handoff.test.sh | 7 +- tests/fm-remote-doctor.test.sh | 181 +++- tests/fm-remote-job.test.sh | 530 ++++++++++ tests/fm-remote-reply.test.sh | 4 +- ...fm-remote-secondmate-lifecycle-e2e.test.sh | 32 +- ...fm-remote-secondmate-trace-context.test.sh | 4 +- 26 files changed, 2935 insertions(+), 297 deletions(-) create mode 100755 bin/fm-remote-job-lib.sh create mode 100755 bin/fm-remote-job-worker.sh create mode 100755 tests/fm-remote-job.test.sh diff --git a/bin/fm-backlog-handoff.sh b/bin/fm-backlog-handoff.sh index d979422202b..e83a857e8a3 100755 --- a/bin/fm-backlog-handoff.sh +++ b/bin/fm-backlog-handoff.sh @@ -309,7 +309,7 @@ remote_deliver_outbox() { # fi rm -f -- "$snapshot" if ! receive_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-backlog-receive.sh \ - "$remote_rel" "$bytes" "$hash" "$generation" 2>&1); then + "$remote_rel" "$bytes" "$hash" "$generation" < /dev/null 2>&1); then [ -z "$receive_out" ] || printf '%s\n' "$receive_out" >&2 echo "error: handoff receipt by $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2 return 1 diff --git a/bin/fm-bootstrap.sh b/bin/fm-bootstrap.sh index ed1f5164984..d3d3bb07898 100755 --- a/bin/fm-bootstrap.sh +++ b/bin/fm-bootstrap.sh @@ -459,7 +459,7 @@ secondmate_sync() { fi nudge_needed=0 converged=1 - if sync_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh sync "$id" 2>&1); then + if sync_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh sync "$id" < /dev/null 2>&1); then case "$sync_out" in synced:*) nudge_needed=1 ;; esac else echo "SECONDMATE_SYNC: secondmate $id: skipped: remote tracked-file sync failed on $remote_host: $(first_line "$sync_out")" @@ -527,7 +527,7 @@ secondmate_liveness_sweep() { echo "SECONDMATE_LIVENESS: secondmate $id: skipped: remote readiness failed on $remote_host: $readiness_reason" continue fi - if out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh state "$id" 2>/dev/null); then + if out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh state "$id" < /dev/null 2>/dev/null); then remote_rc=0 else remote_rc=$? @@ -543,7 +543,7 @@ secondmate_liveness_sweep() { agent_state=$(printf '%s\n' "$out" | tail -1) case "$agent_state" in alive) - if route_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh route "$id" 2>/dev/null); then + if route_out=$("$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh route "$id" < /dev/null 2>/dev/null); then remote_rc=0 else remote_rc=$? diff --git a/bin/fm-fleet-snapshot.sh b/bin/fm-fleet-snapshot.sh index 83e14f87efb..ffa4c639de6 100755 --- a/bin/fm-fleet-snapshot.sh +++ b/bin/fm-fleet-snapshot.sh @@ -481,7 +481,7 @@ task_json_lines() { agent_alive=not_checked if [ -n "$remote_host" ]; then if remote_state=$(run_timed "$FM_SNAPSHOT_SECONDMATE_TIMEOUT" \ - "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh state "$id" 2>/dev/null); then + "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-secondmate-control.sh state "$id" < /dev/null 2>/dev/null); then remote_rc=0 else remote_rc=$? @@ -1200,7 +1200,7 @@ secondmate_current_json() { # if [ -z "$reason" ]; then if [ "$remote" = true ]; then summary=$(run_timed "$FM_SNAPSHOT_SECONDMATE_TIMEOUT" \ - "$SCRIPT_DIR/fm-on.sh" "$id" fm-fleet-snapshot.sh --secondmate-home-summary 2>/dev/null) + "$SCRIPT_DIR/fm-on.sh" "$id" fm-fleet-snapshot.sh --secondmate-home-summary < /dev/null 2>/dev/null) summary_rc=$? else summary=$(run_timed "$FM_SNAPSHOT_SECONDMATE_TIMEOUT" env \ diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index e623a65aacc..3d656f22b07 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -1041,7 +1041,7 @@ fm_pending_reply_tick() { # if [ "$found" = 0 ]; then if [ -n "$remote_host" ]; then observation=$("$_FM_PENDING_REPLY_LIB_DIR/fm-on.sh" "$task_id" \ - fm-remote-secondmate-control.sh observe "$task_id" 2>/dev/null || printf 'unknown') + fm-remote-secondmate-control.sh observe "$task_id" < /dev/null 2>/dev/null || printf 'unknown') case "$observation" in busy|idle|fallback-idle|unknown) ;; *) observation=unknown ;; esac else observation=$(fm_pending_reply_backend_observation "$backend" "$target" "$label" "$harness") diff --git a/bin/fm-procevent-remote-reply.sh b/bin/fm-procevent-remote-reply.sh index 5a0e5b6896c..7518178a400 100755 --- a/bin/fm-procevent-remote-reply.sh +++ b/bin/fm-procevent-remote-reply.sh @@ -194,7 +194,7 @@ cmd_source() { validate_id "$id" read_cursor "$id" exec "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-delta-read.sh \ - "$REMOTE_LOG" "$CURSOR_OFFSET" "$CURSOR_HASH" "$WAIT_SECONDS" + "$REMOTE_LOG" "$CURSOR_OFFSET" "$CURSOR_HASH" "$WAIT_SECONDS" < /dev/null } safe_doc_path() { @@ -219,7 +219,7 @@ fetch_document() { # case "$parent_real" in "$base"|"$base"/*) ;; *) return 1 ;; esac [ ! -L "$destination" ] || return 1 tmp=$(umask 077; mktemp "$parent/.remote-doc.XXXXXX") || return 1 - if ! "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-file.sh get "$rel" "$MAX_DOC_BYTES" > "$tmp"; then + if ! "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-file.sh get "$rel" "$MAX_DOC_BYTES" < /dev/null > "$tmp"; then rm -f -- "$tmp" return 1 fi diff --git a/bin/fm-remote-doctor.sh b/bin/fm-remote-doctor.sh index 4e76c755d72..4ded1557b8a 100755 --- a/bin/fm-remote-doctor.sh +++ b/bin/fm-remote-doctor.sh @@ -4,20 +4,17 @@ # Usage: # bin/fm-on.sh fm-remote-doctor.sh [--fix] # -# Run it through fm-on.sh so the fixed remote entrypoint composes and exports -# the child PATH: this command reports the PATH it inherits instead of -# recomposing it, so the entrypoint stays the single owner of that ordering and -# the two can never drift. +# Run it through fm-on.sh so the fixed entrypoint invokes this readiness owner +# over its plain SSH bootstrap. The command reports the same filesystem-composed +# PATH used by worker jobs while retaining authority to inspect and repair the +# worker itself. # # A remote second mate always runs on the Herdr backend in the dedicated -# fm-remote session, so readiness is more than tool resolution. herdr must -# resolve, that server must be reachable, and on macOS the Firstmate-owned -# launch agent dev.firstmate.herdr.fm-remote at -# ~/Library/LaunchAgents/dev.firstmate.herdr.fm-remote.plist must exist, carry -# LimitLoadToSessionType=Aqua, and be loaded into the console user's gui/ -# domain, so the server belongs to the GUI login session and survives logout and -# SSH disconnection. SSH cannot create an Aqua session, so a host with no GUI -# login is reported as a human gap rather than repaired. +# fm-remote session. Its account therefore needs the Firstmate-owned Aqua Herdr +# agent plus the sibling +# dev.firstmate.remote-job worker that executes every fm-on command in the GUI +# session. SSH cannot create an Aqua session, so a host with no GUI login is a +# human gap rather than something --fix attempts to bypass. # # Line protocol, one fact per line, stable for script consumers: # mode=check|fix @@ -38,12 +35,13 @@ # fixed. Any remaining fixable or human gap, and any missing required tool, # exits non-zero. # -# --fix is idempotent and closes only automatable gaps: it writes the Aqua -# launch agent, bootstraps and kickstarts it into gui/ when a login session -# exists, starts the herdr server where no launch agent applies, and recreates -# the entrypoint symlink. It never creates a login session, never writes an -# auto-login password (kcpassword), never changes FileVault, and never stores an -# account password; those remain reported human gaps. +# --fix is idempotent and closes only automatable gaps: it writes and reloads +# both Firstmate-owned Aqua agents, starts the Linux workers where no Aqua agent +# applies, recreates the entrypoint symlink, and may add an owned ~/.local/bin +# wrapper for a required tool it can discover under nvm, asdf, or mise. It never +# installs packages, creates a login session, writes an auto-login password, +# changes FileVault, stores an account password, or replaces a non-Firstmate +# wrapper; those remain reported gaps. set -eu # Resolve this script's directory with builtins only: a host missing a required @@ -52,8 +50,14 @@ SCRIPT_SELF=${BASH_SOURCE[0]} SCRIPT_DIR=${SCRIPT_SELF%/*} [ "$SCRIPT_DIR" != "$SCRIPT_SELF" ] || SCRIPT_DIR=. SCRIPT_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR" && pwd -P) -REQUIRED_TOOLS=(git jq) -OPTIONAL_TOOLS=(tmux treehouse no-mistakes tasks-axi claude codex opencode pi grok kimi) +FM_ROOT="${FM_ROOT_OVERRIDE:-$(CDPATH='' cd "$SCRIPT_DIR/.." && pwd -P)}" +# shellcheck source=bin/fm-remote-job-lib.sh +. "$SCRIPT_DIR/fm-remote-job-lib.sh" +# shellcheck source=bin/fm-tasks-axi-lib.sh +. "$SCRIPT_DIR/fm-tasks-axi-lib.sh" +REQUIRED_TOOLS=(git jq herdr tasks-axi treehouse) +HARNESS_TOOLS=(claude codex opencode pi pi-signed grok kimi) +OPTIONAL_TOOLS=(tmux no-mistakes gh) LAUNCH_AGENT_LABEL=dev.firstmate.herdr.fm-remote # The dedicated remote-secondmate session. The user's interactive Herdr work # remains in the separate default session, which this readiness check never @@ -71,17 +75,16 @@ MODE=check case "${1:-}" in '') ;; --fix) MODE=fix; shift ;; + --worker-tool-probe) + [ "${FM_REMOTE_JOB_ACTIVE:-}" = 1 ] || { printf 'error: worker tool probe requires the remote job worker\n' >&2; exit 64; } + MODE='worker-tool-probe' + shift + ;; *) usage ;; esac [ "$#" -eq 0 ] || usage -PLATFORM_RAW=$(uname -s 2>/dev/null) || PLATFORM_RAW= -case "$PLATFORM_RAW" in - Darwin) PLATFORM=darwin ;; - Linux) PLATFORM=linux ;; - '') PLATFORM=unknown ;; - *) PLATFORM=$PLATFORM_RAW ;; -esac +PLATFORM=$(fm_remote_job_platform) UID_NUM=$(id -u 2>/dev/null) || UID_NUM= CHECK_NAMES=() @@ -111,8 +114,24 @@ check_is_ok() { # return 1 } +set_check() { # [operator-action] + local i=0 + while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do + if [ "${CHECK_NAMES[$i]}" = "$1" ]; then + CHECK_VALUES[i]=$2 + CHECK_ACTIONS[i]=${3:-} + return 0 + fi + i=$((i + 1)) + done + record "$@" +} + herdr_cli_available() { - command -v herdr >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 + local herdr_bin jq_bin + herdr_bin=$(command -v herdr 2>/dev/null || true) + jq_bin=$(command -v jq 2>/dev/null || true) + [ -n "$herdr_bin" ] && [ -x "$herdr_bin" ] && [ -n "$jq_bin" ] && [ -x "$jq_bin" ] } # The herdr adapter is the single owner of session-scoped herdr invocation and @@ -204,11 +223,235 @@ launch_agent_loaded_contract_matches() { [[ "$loaded" == *'properties=keepalive|runatload'* ]] || return 1 } +# --- remote job and tool checks --------------------------------------------- + +remote_job_existing_state() { + local root + root=${FM_REMOTE_JOB_STATE_ROOT:-${HOME:-}/.firstmate/remote-job} + root=$(fm_remote_job_canonical_existing_dir "$root") || return 1 + fm_remote_job_canonical_existing_dir "$root/jobs" >/dev/null || return 1 + # shellcheck disable=SC2034 # The sourceable worker helpers consume the validated state root. + FM_REMOTE_JOB_STATE=$root +} + +remote_job_probe_ok() { + local ready mtime now + [ "${FM_REMOTE_JOB_ACTIVE:-}" = 1 ] && return 0 + remote_job_existing_state || return 1 + ready="$FM_REMOTE_JOB_STATE/worker.ready" + [ -f "$ready" ] && [ ! -L "$ready" ] || return 1 + mtime=$(fm_remote_job_path_mtime "$ready" 2>/dev/null || true) + case "$mtime" in ''|*[!0-9]*) return 1 ;; esac + now=$(date +%s) + [ $((now - mtime)) -le 10 ] +} + +remote_job_identity_ok() { + [ "${FM_REMOTE_JOB_ACTIVE:-}" = 1 ] && return 0 + remote_job_probe_ok || return 1 + fm_remote_job_worker_identity_matches "$FM_ROOT" "${HOME:-}" +} + +check_remote_job_worker() { + local worker + worker="$FM_ROOT/bin/fm-remote-job-worker.sh" + if [ ! -f "$worker" ] || [ -L "$worker" ] || [ ! -x "$worker" ]; then + record remote-job-worker "human: the configured Firstmate code root has no safe remote job worker" \ + "update the remote Firstmate checkout, then rerun this command with --fix" + record remote-job-worker-loaded "skip: no worker executable is available" + record remote-job-probe "skip: no worker executable is available" + return 0 + fi + if [ "$PLATFORM" = darwin ]; then + fm_remote_job_launchagent_paths "${HOME:-}" + if fm_remote_job_launchagent_contract_matches "$FM_ROOT" "${HOME:-}"; then + record remote-job-worker "ok: $FM_REMOTE_JOB_LAUNCH_AGENT_PLIST matches the Firstmate-owned Aqua worker contract" + else + record remote-job-worker "fixable: $FM_REMOTE_JOB_LAUNCH_AGENT_PLIST does not match the Firstmate-owned Aqua worker contract" \ + "rerun this command with --fix to write dev.firstmate.remote-job" + fi + if [ -z "$UID_NUM" ] || ! command -v launchctl >/dev/null 2>&1; then + record remote-job-worker-loaded "human: the remote job worker cannot be inspected without launchctl and an account uid" \ + "restore launchctl and a readable account uid, then rerun this command" + elif fm_remote_job_launchagent_loaded "$FM_ROOT" "${HOME:-}" "$UID_NUM"; then + record remote-job-worker-loaded "ok: $FM_REMOTE_JOB_LABEL is loaded in gui/$UID_NUM" + elif check_is_ok gui-session; then + record remote-job-worker-loaded "fixable: $FM_REMOTE_JOB_LABEL is not loaded in gui/$UID_NUM" \ + "rerun this command with --fix to bootstrap the worker" + else + record remote-job-worker-loaded "human: $FM_REMOTE_JOB_LABEL cannot be loaded because gui/$UID_NUM has no login session" \ + "close the login-session gap first; SSH cannot create an Aqua session" + fi + else + local pid + pid=$(cat "${FM_REMOTE_JOB_STATE_ROOT:-${HOME:-}/.firstmate/remote-job}/worker.pid" 2>/dev/null || true) + if [ "${FM_REMOTE_JOB_ACTIVE:-}" = 1 ] || + { remote_job_existing_state && case "$pid" in ''|*[!0-9]*) false ;; *) kill -0 "$pid" 2>/dev/null ;; esac; }; then + record remote-job-worker "ok: the Linux remote job worker is running" + record remote-job-worker-loaded "skip: Aqua launch agents do not apply on $PLATFORM" + else + record remote-job-worker "fixable: the Linux remote job worker is not running" \ + "rerun this command with --fix to start it" + record remote-job-worker-loaded "skip: Aqua launch agents do not apply on $PLATFORM" + fi + fi + if ! remote_job_probe_ok; then + record remote-job-probe "fixable: the remote job worker has not reported a fresh probe" \ + "rerun this command with --fix to restart the worker, then rerun through fm-on.sh" + elif ! remote_job_identity_ok; then + set_check remote-job-worker "fixable: the running remote job worker does not match the current Firstmate code" \ + "rerun this command with --fix to reload the current worker" + record remote-job-probe "fixable: the remote job worker identity is stale, so its runtime cannot be probed" \ + "rerun this command with --fix to reload the current worker" + else + record remote-job-probe "ok: the remote job worker published a fresh heartbeat" + fi +} + +report_required_tools() { + local tool resolved harness + MISSING=() + for tool in "${REQUIRED_TOOLS[@]}"; do + resolved=$(command -v "$tool" 2>/dev/null || true) + if [ -n "$resolved" ] && [ -x "$resolved" ]; then + if [ "$tool" = tasks-axi ] && ! fm_tasks_axi_compatible; then + printf 'required tasks-axi=MISSING (incompatible)\n' + MISSING+=(tasks-axi) + else + printf 'required %s=%s\n' "$tool" "$resolved" + fi + else + printf 'required %s=MISSING\n' "$tool" + MISSING+=("$tool") + fi + done + for harness in "${HARNESS_TOOLS[@]}"; do + resolved=$(command -v "$harness" 2>/dev/null || true) + if [ -n "$resolved" ] && [ -x "$resolved" ]; then + printf 'required harness=%s:%s\n' "$harness" "$resolved" + return 0 + fi + done + printf 'required harness=MISSING\n' + MISSING+=(harness) +} + +report_required_tools_from_worker() { + local job_id probe_stdout probe_stderr probe_exit line fact name value + local expected=6 count=0 valid=1 seen=' ' + if ! job_id=$(fm_remote_job_stage "${HOME:-}" "$FM_ROOT" "${FM_HOME:-}" \ + fm-remote-doctor.sh --worker-tool-probe /dev/null || true + set_check remote-job-probe "fixable: the remote job worker did not complete the required-tool probe" \ + "rerun this command with --fix to restart the worker" + report_required_tools + return 0 + fi + probe_stdout=$FM_REMOTE_JOB_STDOUT + probe_stderr=$FM_REMOTE_JOB_STDERR + probe_exit=$FM_REMOTE_JOB_EXIT + MISSING=() + while IFS= read -r line; do + case "$line" in required\ *=*) ;; *) valid=0; continue ;; esac + fact=${line#required } + name=${fact%%=*} + value=${fact#*=} + case "$name" in git|jq|herdr|tasks-axi|treehouse|harness) ;; *) valid=0; continue ;; esac + case "$seen" in *" $name "*) valid=0; continue ;; esac + seen="$seen$name " + count=$((count + 1)) + case "$value" in MISSING*) MISSING+=("$name") ;; '') valid=0 ;; esac + done < "$probe_stdout" + [ "$count" -eq "$expected" ] || valid=0 + [ ! -s "$probe_stderr" ] || valid=0 + case "$probe_exit:${#MISSING[@]}" in 0:0|1:[1-9]*) ;; *) valid=0 ;; esac + if [ "$valid" -eq 1 ]; then + cat "$probe_stdout" + set_check remote-job-probe "ok: the remote job worker completed the required-tool probe" + else + set_check remote-job-probe "fixable: the remote job worker returned an invalid required-tool probe result" \ + "rerun this command with --fix to restart the worker" + report_required_tools + fi + fm_remote_job_reap "${HOME:-}" "$job_id" 2>/dev/null || true +} + +wrapper_is_firstmate_owned() { # + local path=$1 first second + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + IFS= read -r first < "$path" || return 1 + IFS= read -r second < <(tail -n +2 "$path") || return 1 + [ "$first" = '#!/usr/bin/env bash' ] && [ "$second" = '# Firstmate remote tool wrapper v1' ] +} + +repair_tool_wrapper() { # + local tool=$1 target wrapper tmp + local resolved + resolved=$(command -v "$tool" 2>/dev/null || true) + [ -n "$resolved" ] && [ -x "$resolved" ] && return 0 + target=$(fm_remote_job_manager_tool "${HOME:-}" "$tool" 2>/dev/null || true) + [ -n "$target" ] || return 1 + wrapper="${HOME:-}/.local/bin/$tool" + if [ -e "$wrapper" ] || [ -L "$wrapper" ]; then + if ! wrapper_is_firstmate_owned "$wrapper"; then + fix_report "required-$tool" failed "$wrapper exists and is not Firstmate-owned" + return 1 + fi + else + if ! mkdir -p "${HOME:-}/.local/bin" 2>/dev/null || [ -L "${HOME:-}/.local/bin" ]; then + fix_report "required-$tool" failed "cannot create ${HOME:-}/.local/bin" + return 1 + fi + fi + tmp="${HOME:-}/.local/bin/.$tool.tmp.$$" + { + printf '%s\n' '#!/usr/bin/env bash' + printf '%s\n' '# Firstmate remote tool wrapper v1' + printf 'exec %q "$@"\n' "$target" + } > "$tmp" || { rm -f -- "$tmp"; fix_report "required-$tool" failed "cannot write $wrapper"; return 1; } + if ! chmod 0700 "$tmp" || ! mv -f -- "$tmp" "$wrapper"; then + rm -f -- "$tmp" + fix_report "required-$tool" failed "cannot publish $wrapper" + return 1 + fi + fix_report "required-$tool" applied "linked the discoverable version-manager tool at $wrapper" +} + +repair_required_wrappers() { + local tool resolved + for tool in "${REQUIRED_TOOLS[@]}"; do + repair_tool_wrapper "$tool" || true + done + for tool in "${HARNESS_TOOLS[@]}"; do + resolved=$(command -v "$tool" 2>/dev/null || true) + [ -z "$resolved" ] || [ ! -x "$resolved" ] || return 0 + done + for tool in "${HARNESS_TOOLS[@]}"; do + fm_remote_job_manager_tool "${HOME:-}" "$tool" >/dev/null 2>&1 || continue + repair_tool_wrapper "$tool" && return 0 + done +} + +fix_remote_job_worker() { + if fm_remote_job_ensure_worker "$FM_ROOT" "${HOME:-}"; then + [ "$FM_REMOTE_JOB_REPAIRED" -eq 0 ] || fix_report remote-job-worker applied "installed or reloaded $FM_REMOTE_JOB_LABEL" + return 0 + fi + fix_report remote-job-worker failed "${FM_REMOTE_JOB_ERROR:-the remote job worker could not start}" + return 1 +} + # --- checks ----------------------------------------------------------------- check_herdr() { local resolved - if resolved=$(command -v herdr 2>/dev/null); then + if resolved=$(command -v herdr 2>/dev/null) && [ -x "$resolved" ]; then record herdr "ok: $resolved" return 0 fi @@ -336,6 +579,7 @@ run_checks() { CHECK_ACTIONS=() check_herdr check_gui_session + check_remote_job_worker check_launch_agent check_herdr_server check_entrypoint_link @@ -441,7 +685,8 @@ link_entrypoint() { } apply_fixes() { - local i name value launch_agent_written=0 launch_agent_reloaded=0 + local i name value launch_agent_written=0 launch_agent_reloaded=0 remote_job_fixed=0 + repair_required_wrappers i=0 while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do name=${CHECK_NAMES[$i]} @@ -449,6 +694,11 @@ apply_fixes() { i=$((i + 1)) case "$value" in fixable:*) ;; *) continue ;; esac case "$name" in + remote-job-worker|remote-job-worker-loaded|remote-job-probe) + [ "$remote_job_fixed" -eq 0 ] || continue + remote_job_fixed=1 + fix_remote_job_worker || true + ;; launchagent|launchagent-scope) [ "$launch_agent_written" -eq 0 ] || continue launch_agent_written=1 @@ -483,6 +733,12 @@ apply_fixes() { # --- report ----------------------------------------------------------------- +if [ "$MODE" = worker-tool-probe ]; then + report_required_tools + [ "${#MISSING[@]}" -eq 0 ] + exit +fi + printf 'mode=%s\n' "$MODE" printf 'path=%s\n' "${PATH:-}" if [ -n "${FM_ROOT_OVERRIDE:-}" ] && [ "${PATH%%:*}" = "$FM_ROOT_OVERRIDE/bin" ]; then @@ -493,23 +749,6 @@ else fi printf 'platform=%s\n' "$PLATFORM" -MISSING=() -for tool in "${REQUIRED_TOOLS[@]}"; do - if resolved=$(command -v "$tool" 2>/dev/null); then - printf 'required %s=%s\n' "$tool" "$resolved" - else - printf 'required %s=MISSING\n' "$tool" - MISSING+=("$tool") - fi -done -for tool in "${OPTIONAL_TOOLS[@]}"; do - if resolved=$(command -v "$tool" 2>/dev/null); then - printf 'optional %s=%s\n' "$tool" "$resolved" - else - printf 'optional %s=absent\n' "$tool" - fi -done - run_checks if [ "$MODE" = fix ]; then apply_fixes @@ -518,6 +757,19 @@ if [ "$MODE" = fix ]; then run_checks fi +if [ "${FM_REMOTE_JOB_ACTIVE:-}" = 1 ] || ! remote_job_identity_ok; then + report_required_tools +else + report_required_tools_from_worker +fi +for tool in "${OPTIONAL_TOOLS[@]}"; do + if resolved=$(command -v "$tool" 2>/dev/null); then + printf 'optional %s=%s\n' "$tool" "$resolved" + else + printf 'optional %s=absent\n' "$tool" + fi +done + GAPS=() i=0 while [ "$i" -lt "${#CHECK_NAMES[@]}" ]; do @@ -534,7 +786,7 @@ done if [ "${#MISSING[@]}" -gt 0 ]; then printf 'error: required tools do not resolve on the remote runtime PATH: %s\n' "${MISSING[*]}" >&2 printf 'fix: install each one where it resolves on the path reported above, or put a wrapper script for it in %s/.local/bin, which is always on that PATH.\n' "${HOME:-~}" >&2 - printf 'fix: tools provided by nvm, asdf, or mise never resolve here because no login or interactive shell runs; see docs/remote-secondmates.md for the wrapper recipe.\n' >&2 + printf 'fix: tools in an unselected nvm version or outside the discovered asdf or mise paths need an absolute wrapper; see docs/remote-secondmates.md for the wrapper recipe.\n' >&2 fi if [ "${#MISSING[@]}" -gt 0 ] || [ "${#GAPS[@]}" -gt 0 ]; then NAMES= diff --git a/bin/fm-remote-entrypoint.sh b/bin/fm-remote-entrypoint.sh index f977511de92..6a9a188a76d 100755 --- a/bin/fm-remote-entrypoint.sh +++ b/bin/fm-remote-entrypoint.sh @@ -1,138 +1,49 @@ #!/usr/bin/env bash # Fixed remote entrypoint for bin/fm-on.sh. # -# Install this tracked file as `fm-remote-entrypoint.sh` on the remote account's -# non-interactive SSH PATH. It accepts only protocol metadata and encoded argv -# from fm-on.sh, validates the configured code root and FM_HOME on this host, -# resolves one genuine non-symlink executable under /bin/fm-*.sh, then -# executes it directly. It never accepts a shell command string. +# Install this tracked file as fm-remote-entrypoint.sh on the remote account's +# non-interactive SSH PATH. It accepts protocol metadata plus a base64-encoded +# NUL argv stream, validates one genuine tracked executable in /bin/fm-*.sh, +# then stages it for the Firstmate-owned remote job worker. It never accepts a +# shell command string. # -# Protocol v1 argv is a base64-encoded NUL-delimited stream. stdin is not used -# for protocol framing, so it remains byte-for-byte available to the command. -# The child receives an empty environment plus fixed PATH, HOME, FM_HOME, and -# FM_ROOT_OVERRIDE. stdout, stderr, and exit status pass through unchanged. +# The readiness-owning fm-remote-doctor.sh runs in this plain SSH bootstrap so +# check mode can inspect worker gaps without changing them and --fix can repair +# them. Every other command is staged after the worker is ready. On Darwin, a +# missing Aqua session fails before staging with the doctor-actionable +# console-login diagnostic. Linux uses the same queue and worker shape without +# an Aqua requirement. # -# This file is the single owner of the child PATH. compose_operator_path builds -# its operator portion from the account's ~/.local/bin, the common -# package-manager directories that exist on this host, and the always-present -# system tail. The tracked-command check resolves git from that portion before -# /bin is prepended for the child. No login or interactive shell is ever -# started, so a tool that -# lives outside those directories - anything installed by nvm, asdf, or mise - -# needs a wrapper in ~/.local/bin. bin/fm-remote-doctor.sh reports this exact -# PATH by inheriting it rather than recomposing it, so the two cannot drift. +# stdin is captured as bounded job input. The completed worker result is relayed +# with stdout and stderr kept separate and its exit status preserved. An SSH +# disconnect remains unknown completion to fm-on.sh, which preserves OpenSSH's +# exit 255 behavior. The shared library header owns job fields, bounds, PATH, +# LaunchAgent contract, and worker environment. set -eu PROTOCOL=1 -OPERATOR_PATH= +DOCTOR_SHA256=8746014c53bd5f195dd43444369b387e3fdb2b7d8f120b06e8f3c2a86ee31c3a +SCRIPT_DIR=$(CDPATH='' cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) -die() { printf 'error: %s\n' "$1" >&2; exit "${2:-64}"; } - -path_append() { # - case ":$OPERATOR_PATH:" in *":$1:"*) return 0 ;; esac - OPERATOR_PATH="${OPERATOR_PATH:+$OPERATOR_PATH:}$1" -} - -path_append_if_dir() { # - [ -d "$1" ] || return 0 - path_append "$1" -} - -build_child_path() { # - local root_bin=$1 directory old_ifs - CHILD_PATH=$root_bin - old_ifs=$IFS - IFS=: - for directory in $OPERATOR_PATH; do - case ":$CHILD_PATH:" in *":$directory:"*) continue ;; esac - CHILD_PATH="$CHILD_PATH:$directory" - done - IFS=$old_ifs -} +# shellcheck source=bin/fm-remote-job-lib.sh +. "$SCRIPT_DIR/fm-remote-job-lib.sh" -compose_operator_path() { # - local account_home=$1 account_user - OPERATOR_PATH= - path_append "$account_home/.local/bin" - path_append_if_dir "$account_home/.nix-profile/bin" - # env -i clears USER, so ask the password database rather than the environment. - account_user=$(id -un 2>/dev/null) || account_user= - if [ -n "$account_user" ]; then - path_append_if_dir "/etc/profiles/per-user/$account_user/bin" - fi - path_append_if_dir /run/current-system/sw/bin - path_append_if_dir /opt/homebrew/bin - path_append_if_dir /usr/local/bin - path_append /usr/bin - path_append /bin - path_append /usr/sbin - path_append /sbin -} +die() { printf 'error: %s\n' "$1" >&2; exit "${2:-64}"; } base64_decode_to() { # local encoded=$1 destination=$2 - if printf '%s' "$encoded" | base64 --decode > "$destination" 2>/dev/null; then - return 0 - fi - if printf '%s' "$encoded" | base64 -D > "$destination" 2>/dev/null; then - return 0 - fi + if printf '%s' "$encoded" | base64 --decode > "$destination" 2>/dev/null; then return 0; fi + if printf '%s' "$encoded" | base64 -D > "$destination" 2>/dev/null; then return 0; fi return 1 } decode_text() { #