diff --git a/.agents/skills/afk/SKILL.md b/.agents/skills/afk/SKILL.md index 68b7bf1b7cd..3d278cc019b 100644 --- a/.agents/skills/afk/SKILL.md +++ b/.agents/skills/afk/SKILL.md @@ -43,7 +43,7 @@ Hold-for-return is the default and the only reach profile this release records: - **Every other harness** (codex, opencode, omp, and cursor on a home that does not run the supervision host, and kimi): run `bin/fm-afk-launch.sh start`. It is the single owner of the daemon terminal: it creates a NON-VISIBLE tracked terminal for the current backend and passes the captain pane in as `FM_SUPERVISOR_TARGET` so the daemon injects into the captain, not its own new pane (docs/herdr-backend.md "Away-mode supervisor support"). Both daemon paths require the record `enter` wrote and share `bin/fm-afk-start.sh` as the daemon entry. - The daemon is **presence-gated**: it injects escalations only while `state/.afk` exists, and stays quiet otherwise. + The daemon is **presence-gated**: it injects escalations only while `state/.afk` exists, and exits on its own once that flag is gone, so it never takes the attended session's wakes. 3. **Announce, then read back after entry.** Relay the announcement in spirit: hold-for-return only, no phone channel, your instructions are recorded and the away session will carry them out where it can, anything it is unsure of, or that needs you, waits for your return, and destructive, irreversible, and security-sensitive actions are never pre-authorizable whatever the words say. Then give your own plain-sentence restatement of the words in `AGENTS.md` section 9 language - what you read them as asking for, sentence by sentence, never a numbered field list - beside the expected return, the spend cap, and the one-sentence reach announcement. diff --git a/bin/fm-afk-daemon-lib.sh b/bin/fm-afk-daemon-lib.sh new file mode 100755 index 00000000000..e5731c7c7b5 --- /dev/null +++ b/bin/fm-afk-daemon-lib.sh @@ -0,0 +1,215 @@ +#!/usr/bin/env bash +# fm-afk-daemon-lib.sh - the single owner of finding a home's live away-mode +# supervise daemons (bin/fm-supervise-daemon.sh): its singleton lock and the +# parent link of the watcher a daemon runs. Sourced by the daemon, +# bin/fm-afk-start.sh, bin/fm-afk-launch.sh, bin/fm-watch-arm.sh, and +# bin/fm-watch-checkpoint.sh, each after bin/fm-wake-lib.sh, whose fm_pid_alive, +# fm_pid_identity, fm_pid_identity_matches, fm_watcher_lock_matches_pid, +# fm_path_mtime, and fm_epoch_seconds_to it calls. Only fm_afk_daemon_stop +# signals anything. +# +# The lock alone cannot find every live daemon: a daemon can outlive its lock, +# and a recorded identity can stop matching its live process. No host time-zone +# change moves an identity (bin/fm-pid-identity-lib.sh), and the match bridges a +# record a build before that fix wrote in local time, but such a record written +# under another locale, or a wall-clock step on a host without /proc, still +# leaves a live daemon unmatched by its lock and its watcher by the watcher lock. +# The watcher proof finds such a daemon while it still runs this home's watcher; +# a daemon that runs neither exits on its own once state/.afk is gone or its +# lock is lost. +# +# Proofs that a pid is a live daemon of this home, strongest first: +# lock the lock names a live pid whose recorded identity still matches, +# or that recorded no identity and runs the daemon script. +# watcher this home's watcher's live parent process runs the daemon script, +# which also names a daemon without its lock. The watcher is the +# watcher lock's pid, verified by its recorded identity or, once that +# identity drifted, by the lock's home and watch path, a live command +# line that runs the watch path, and a start no later than the lock's +# pid-identity file was written. +# command the lock names a live pid that runs the daemon script although its +# recorded identity no longer matches. That keeps the lock and stops a +# second daemon from starting; it is never enough to signal the pid. + +FM_AFK_DAEMON_SCRIPT=fm-supervise-daemon.sh + +fm_afk_daemon_lock_path() { # + printf '%s/.supervise-daemon.lock\n' "$1" +} + +# Print 's command line, from a Linux-compatible proc root when present. +fm_afk_daemon_pid_command() { # + local pid=$1 proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + if [ -r "$proc_root/$pid/cmdline" ]; then + tr '\0' ' ' < "$proc_root/$pid/cmdline" 2>/dev/null + return + fi + COLUMNS=10000 LC_ALL=C ps -p "$pid" -o command= 2>/dev/null +} + +# Print 's parent pid. +fm_afk_daemon_pid_parent() { # + local pid=$1 proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} stat_line parent + local -a fields + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + if [ -r "$proc_root/$pid/stat" ]; then + stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 + # After the final comm delimiter, index 1 is proc stat field 4 (ppid). + read -r -a fields <<< "${stat_line##*)}" + parent=${fields[1]:-} + else + parent=$(ps -o ppid= -p "$pid" 2>/dev/null | tr -d '[:space:]') + fi + case "$parent" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' "$parent" +} + +# True when is alive and its command line runs the daemon script. +fm_afk_daemon_pid_runs_daemon() { # + local command + fm_pid_alive "$1" || return 1 + command=$(fm_afk_daemon_pid_command "$1") || return 1 + case " $command " in + *"/$FM_AFK_DAEMON_SCRIPT "*|*" $FM_AFK_DAEMON_SCRIPT "*) return 0 ;; + esac + return 1 +} + +# Print the lock's owner directory: the symlink target, or a legacy plain dir. +fm_afk_daemon_lock_owner() { # + local lock owner + lock=$(fm_afk_daemon_lock_path "$1") + if [ -L "$lock" ]; then + owner=$(readlink "$lock" 2>/dev/null) || return 1 + [ -n "$owner" ] || return 1 + case "$owner" in + /*) printf '%s\n' "$owner" ;; + *) printf '%s/%s\n' "$(dirname "$lock")" "$owner" ;; + esac + return 0 + fi + [ -d "$lock" ] || return 1 + printf '%s\n' "$lock" +} + +fm_afk_daemon_lock_pid() { # + local owner + owner=$(fm_afk_daemon_lock_owner "$1") || return 1 + cat "$owner/pid" 2>/dev/null || true +} + +# The lock proof (header): strong enough to signal the lock's pid. +fm_afk_daemon_lock_holder_proven() { # + local state=$1 owner pid identity + owner=$(fm_afk_daemon_lock_owner "$state") || return 1 + pid=$(cat "$owner/pid" 2>/dev/null || true) + fm_pid_alive "$pid" || return 1 + identity=$(cat "$owner/pid-identity" 2>/dev/null || true) + if [ -z "$identity" ]; then + fm_afk_daemon_pid_runs_daemon "$pid" + return + fi + fm_pid_identity_matches "$pid" "$identity" || return 1 +} + +# The lock or command proof (header): a live daemon holds the lock, so it must +# be kept and no second daemon may start. +fm_afk_daemon_lock_holder_live() { # + local pid + fm_afk_daemon_lock_holder_proven "$1" && return 0 + pid=$(fm_afk_daemon_lock_pid "$1") || return 1 + fm_afk_daemon_pid_runs_daemon "$pid" +} + +# Print 's start time in epoch seconds: now minus its ps elapsed time, +# which, unlike lstart, does not depend on the time zone. Reading now first +# can only move the start earlier. +fm_afk_daemon_pid_start_epoch() { # + local pid=$1 etime days=0 hours=0 minutes=0 seconds now + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + fm_epoch_seconds_to now + etime=$(LC_ALL=C ps -p "$pid" -o etime= 2>/dev/null | tr -d '[:space:]') + case "$etime" in *-*) days=${etime%%-*}; etime=${etime#*-} ;; esac + case "$etime" in *:*:*) hours=${etime%%:*}; etime=${etime#*:} ;; esac + case "$etime" in *:*) minutes=${etime%%:*}; seconds=${etime#*:} ;; *) return 1 ;; esac + case "$days$hours$minutes$seconds" in ''|*[!0-9]*) return 1 ;; esac + printf '%s\n' $(( now - ((10#$days * 24 + 10#$hours) * 60 + 10#$minutes) * 60 - 10#$seconds )) +} + +# True when is the watcher lock's pid for and although +# its recorded identity no longer matches: its live command line runs the watch +# path, and it started no later than the lock's pid-identity file was written, +# with two seconds of slack for the elapsed-time rounding, so a later process +# that reused the pid does not pass. +fm_afk_daemon_watcher_drifted() { # + local lockdir="$1/.watch.lock" watch_path=$2 pid=$3 home=$4 command written started + [ "$(cat "$lockdir/fm-home" 2>/dev/null || true)" = "$home" ] || return 1 + [ "$(cat "$lockdir/watcher-path" 2>/dev/null || true)" = "$watch_path" ] || return 1 + command=$(fm_afk_daemon_pid_command "$pid") || return 1 + case " $command " in *" $watch_path "*) ;; *) return 1 ;; esac + written=$(fm_path_mtime "$lockdir/pid-identity") || return 1 + started=$(fm_afk_daemon_pid_start_epoch "$pid") || return 1 + [ "$started" -le $(( written + 2 )) ] +} + +# The watcher proof (header): print the pid of the daemon whose child is this +# home's live watcher. +fm_afk_daemon_watcher_owner() { # + local state=$1 watch_path=$2 home=$3 watcher parent + watcher=$(cat "$state/.watch.lock/pid" 2>/dev/null || true) + fm_pid_alive "$watcher" || return 1 + fm_watcher_lock_matches_pid "$state" "$watch_path" "$watcher" "$home" \ + || fm_afk_daemon_watcher_drifted "$state" "$watch_path" "$watcher" "$home" \ + || return 1 + parent=$(fm_afk_daemon_pid_parent "$watcher") || return 1 + case "$parent" in 0|1) return 1 ;; esac + fm_afk_daemon_pid_runs_daemon "$parent" || return 1 + printf '%s\n' "$parent" +} + +# Print each live daemon of that a lock or watcher proof names, once. +# The watcher proof needs and . +fm_afk_daemon_live_pids() { # [ ] + local state=$1 watch_path=${2:-} home=${3:-} pid found=' ' + if fm_afk_daemon_lock_holder_proven "$state"; then + pid=$(fm_afk_daemon_lock_pid "$state") + found="$found$pid " + fi + if [ -n "$watch_path" ] && pid=$(fm_afk_daemon_watcher_owner "$state" "$watch_path" "$home"); then + case "$found" in *" $pid "*) ;; *) found="$found$pid " ;; esac + fi + for pid in $found; do + printf '%s\n' "$pid" + done +} + +# SIGTERM each and wait up to ten seconds for all of them to exit. A pid +# that still runs under the identity it had before the signal is printed, and +# then the call fails; a pid that exited or was reused counts as stopped. +fm_afk_daemon_stop() { # ... + local pid i alive status=0 + local -a pids=() identities=() + for pid in "$@"; do + fm_pid_alive "$pid" || continue + pids+=("$pid") + identities+=("$(fm_pid_identity "$pid" 2>/dev/null || true)") + kill -TERM "$pid" 2>/dev/null || true + done + [ "${#pids[@]}" -gt 0 ] || return 0 + for _ in $(seq 1 40); do + alive=0 + for pid in "${pids[@]}"; do + fm_pid_alive "$pid" && alive=1 + done + [ "$alive" -eq 1 ] || break + sleep 0.25 + done + for i in "${!pids[@]}"; do + fm_pid_alive "${pids[$i]}" || continue + [ "$(fm_pid_identity "${pids[$i]}" 2>/dev/null || true)" = "${identities[$i]}" ] || continue + printf '%s\n' "${pids[$i]}" + status=1 + done + return "$status" +} diff --git a/bin/fm-afk-launch.sh b/bin/fm-afk-launch.sh index 23bab8d4da8..05fe3556ac5 100755 --- a/bin/fm-afk-launch.sh +++ b/bin/fm-afk-launch.sh @@ -82,9 +82,12 @@ # fm-afk-launch.sh start-native # Prepare lifecycle state for a harness-native # background job and record that no terminal exists. -# fm-afk-launch.sh stop Correct-ordered exit: SIGTERM the daemon so its -# cleanup flushes WHILE state/.afk is still present, -# wait for it, close a recorded non-native terminal +# fm-afk-launch.sh stop Correct-ordered exit: SIGTERM every live daemon of +# this home, including one that lost its lock but +# still runs this home's watcher +# (bin/fm-afk-daemon-lib.sh), so its cleanup flushes +# WHILE state/.afk is still present, wait for each, +# close a recorded non-native terminal # by exact id, clear state/.afk, then archive the # record last. A Pi or native entry that never # launched a daemon reports that none was running. @@ -164,12 +167,11 @@ FM_AFK_CONTRACT_CMD="$FM_AFK_LAUNCH_DIR/fm-afk-contract.sh" fm_afk_launch_log() { printf 'fm-afk-launch: %s\n' "$*" >&2; } fm_afk_launch_lock_owned() { - local pid expected actual + local pid expected [ -d "$FM_AFK_LAUNCH_LOCK" ] || return 1 pid=$(cat "$FM_AFK_LAUNCH_LOCK/pid" 2>/dev/null) || return 1 expected=$(cat "$FM_AFK_LAUNCH_LOCK/pid-identity" 2>/dev/null) || return 1 - actual=$(fm_pid_identity "$pid" 2>/dev/null) || return 1 - [ -n "$expected" ] && [ "$actual" = "$expected" ] + [ -n "$expected" ] && fm_pid_identity_matches "$pid" "$expected" } fm_afk_launch_lock_acquire() { @@ -813,41 +815,30 @@ fm_afk_launch_start_native() { } fm_afk_launch_stop() { - local pid pid_identity current_identity result=0 read_result archived closed_daemon_terminal=0 + local pid survivors result=0 read_result archived closed_daemon_terminal=0 + local -a daemon_pids=() fm_afk_launch_record_read read_result=$? if [ "$read_result" -eq 2 ]; then fm_afk_launch_log "malformed daemon terminal record; refusing to stop away mode" return 1 fi - # (1) SIGTERM the daemon so its cleanup trap flushes buffered escalations - # WHILE state/.afk is still present (the exit-ordering fix: clearing .afk - # first would make that flush a no-op via inject_msg's presence gate). - pid="" - pid_identity="" - if daemon_lock_held_by_live_daemon; then - pid=$(daemon_lock_pid 2>/dev/null) || return 1 - pid_identity=$(fm_pid_identity "$pid" 2>/dev/null) || return 1 + # (1) SIGTERM every live daemon of this home so each cleanup trap flushes + # buffered escalations WHILE state/.afk is still present (the exit-ordering + # fix: clearing .afk first would make that flush a no-op via inject_msg's + # presence gate). bin/fm-afk-daemon-lib.sh owns which proofs name a daemon, + # including one that lost its lock. + while IFS= read -r pid; do + [ -n "$pid" ] && daemon_pids+=("$pid") + done </dev/null; then - fm_afk_launch_log "failed to signal away-mode daemon pid=$pid" - result=1 - fi - for _ in $(seq 1 40); do - fm_pid_alive "$pid" || break - sleep 0.25 - done - fi - if [ -n "$pid" ] && fm_pid_alive "$pid"; then - current_identity=$(fm_pid_identity "$pid" 2>/dev/null) || { - fm_afk_launch_log "could not confirm away-mode daemon exit; preserving lifecycle state" - return 1 - } - if [ "$current_identity" = "$pid_identity" ]; then - fm_afk_launch_log "away-mode daemon did not exit after SIGTERM; preserving lifecycle state" - return 1 - fi + if daemon_lock_held_by_live_daemon; then + fm_afk_launch_log "a live away daemon pid=$(daemon_lock_pid) still holds the lock, but nothing proves it is this home's, so it was not signalled; the attended arm stops it once it runs this home's watcher" fi # (2) Close the daemon's own terminal by exact id. A native/none record or # an absent record means no terminal existed for this entry (Pi never diff --git a/bin/fm-afk-start.sh b/bin/fm-afk-start.sh index 3ddafc5107f..6a1862881fa 100755 --- a/bin/fm-afk-start.sh +++ b/bin/fm-afk-start.sh @@ -41,6 +41,8 @@ FM_AFK_DAEMON="$FM_AFK_START_DIR/fm-supervise-daemon.sh" # shellcheck source=bin/fm-wake-lib.sh . "$FM_AFK_START_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-afk-daemon-lib.sh +. "$FM_AFK_START_DIR/fm-afk-daemon-lib.sh" fm_afk_start_usage() { sed -n '2,14p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' @@ -67,48 +69,14 @@ fm_afk_clear_stale_artifacts() { # "$state/.subsuper-unknown-acked" 2>/dev/null } -daemon_lock_owner() { - local owner - if [ -L "$FM_AFK_LOCK" ]; then - owner=$(readlink "$FM_AFK_LOCK" 2>/dev/null) || return 1 - [ -n "$owner" ] || return 1 - case "$owner" in - /*) printf '%s\n' "$owner" ;; - *) printf '%s/%s\n' "$(dirname "$FM_AFK_LOCK")" "$owner" ;; - esac - return 0 - fi - [ -d "$FM_AFK_LOCK" ] || return 1 - printf '%s\n' "$FM_AFK_LOCK" -} - -daemon_pid_matches() { - local pid=$1 owner=$2 identity current command - identity=$(cat "$owner/pid-identity" 2>/dev/null || true) - if [ -n "$identity" ]; then - current=$(fm_pid_identity "$pid") || return 1 - [ "$current" = "$identity" ] - return - fi - command=$(ps -p "$pid" -o command= 2>/dev/null || true) - case "$command" in - *"$FM_AFK_DAEMON"*|*"fm-supervise-daemon.sh"*) return 0 ;; - esac - return 1 -} - daemon_lock_pid() { - local owner - owner=$(daemon_lock_owner) || return 1 - cat "$owner/pid" 2>/dev/null || true + fm_afk_daemon_lock_pid "$FM_AFK_STATE" } +# A live daemon holds the lock (bin/fm-afk-daemon-lib.sh's lock or command +# proof): keep the lock and start no second daemon. daemon_lock_held_by_live_daemon() { - local owner pid - owner=$(daemon_lock_owner) || return 1 - pid=$(cat "$owner/pid" 2>/dev/null || true) - fm_pid_alive "$pid" || return 1 - daemon_pid_matches "$pid" "$owner" + fm_afk_daemon_lock_holder_live "$FM_AFK_STATE" } fm_afk_flag_write() { # [mode] @@ -164,6 +132,7 @@ fm_afk_start_main() { return 0 fi + # A live lock pid that is not a daemon was reused, so the lock is stale. if fm_pid_alive "$pid" && [ -n "$pid" ]; then fm_lock_remove_path "$FM_AFK_LOCK" 2>/dev/null || true fi diff --git a/bin/fm-extension.mjs b/bin/fm-extension.mjs index d689e70b257..29bfede3e6d 100755 --- a/bin/fm-extension.mjs +++ b/bin/fm-extension.mjs @@ -77,6 +77,7 @@ import { createHash, randomBytes } from "node:crypto"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { TextDecoder, promisify } from "node:util"; +import { legacyIdentityMatches, lstartParts } from "./fm-pid-identity.mjs"; const SELF = fileURLToPath(import.meta.url); const CODE_ROOT = path.dirname(path.dirname(SELF)); @@ -913,7 +914,8 @@ async function sleep(milliseconds) { async function capturedProcessOutput(command, args, maxBytes = 8192) { const child = spawn(command, args, { - env: { PATH: sanitizedPath(), LANG: "C", LC_ALL: "C" }, + // TZ=UTC0 keeps a ps lstart read independent of the host time zone. + env: { PATH: sanitizedPath(), LANG: "C", LC_ALL: "C", TZ: "UTC0" }, shell: false, stdio: ["ignore", "pipe", "ignore"], }); @@ -933,6 +935,8 @@ async function capturedProcessOutput(command, args, maxBytes = 8192) { return Buffer.concat(chunks).toString("utf8").trim(); } +// Mirrors fm_pid_identity in bin/fm-pid-identity-lib.sh, which owns the +// format: the shell records a claim identity that this host compares. async function pidIdentity(pid) { if (process.platform === "linux") { const stat = await readFile(`/proc/${pid}/stat`, "utf8").catch(() => fail("process-identity-uncertain", "cannot inspect extension process identity")); @@ -944,7 +948,19 @@ async function pidIdentity(pid) { } return `linux-starttime=${fields[19]} cmdline-hex=${cmdline.toString("hex")}`; } - return capturedProcessOutput("/bin/ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="]); + return `lstart-utc=${await capturedProcessOutput("/bin/ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="])}`; +} + +// Mirrors fm_pid_identity_matches: true while pid is still the process the +// recorded identity names. Throws when the identity cannot be read. +async function pidIdentityMatches(pid, expected) { + const actual = await pidIdentity(pid); + if (actual === expected) return true; + if (!lstartParts(expected)) return false; + const utc = actual.startsWith("lstart-utc=") + ? actual.slice("lstart-utc=".length) + : await capturedProcessOutput("/bin/ps", ["-p", String(pid), "-o", "lstart=", "-o", "command="]); + return legacyIdentityMatches(expected, utc); } async function selfIdentity() { @@ -977,13 +993,13 @@ async function processIdentityState(pid, expected) { return pidAlive(pid) ? 2 : 1; } } - let actual; + let matches; try { - actual = await pidIdentity(pid); + matches = await pidIdentityMatches(pid, expected); } catch { return pidAlive(pid) ? 2 : 1; } - return actual === expected ? 0 : 2; + return matches ? 0 : 2; } async function barrierProcessGroupState(pid, expectedIdentity) { @@ -1011,13 +1027,13 @@ async function processGroupState(pid, expectedIdentity = null, trustedChild = fa if (!pidAlive(pid)) return groupAlive(pid) ? 3 : 1; if (expectedIdentity?.startsWith("barrier-token:")) return barrierProcessGroupState(pid, expectedIdentity); if (expectedIdentity) { - let actual; + let matches; try { - actual = await pidIdentity(pid); + matches = await pidIdentityMatches(pid, expectedIdentity); } catch { return pidAlive(pid) ? 2 : (groupAlive(pid) ? 3 : 1); } - if (actual !== expectedIdentity) return 2; + if (!matches) return 2; } else if (!trustedChild) { return 2; } diff --git a/bin/fm-herdr-lab-viewer.py b/bin/fm-herdr-lab-viewer.py index ce40e0b4a9e..2e4bba34d07 100755 --- a/bin/fm-herdr-lab-viewer.py +++ b/bin/fm-herdr-lab-viewer.py @@ -84,7 +84,7 @@ def _process_start(pid): check=True, capture_output=True, text=True, - env={**os.environ, "LC_ALL": "C"}, + env={**os.environ, "LC_ALL": "C", "TZ": "UTC0"}, ) value = result.stdout.strip() if not value: @@ -98,9 +98,9 @@ def _write_pidfile(path, launcher_pid, viewer_pid): temporary = "%s.%d.tmp" % (path, launcher_pid) with open(temporary, "w", encoding="utf-8") as handle: handle.write("launcher_pid=%d\n" % launcher_pid) - handle.write("launcher_start=%s\n" % launcher_start) + handle.write("launcher_start_utc=%s\n" % launcher_start) handle.write("viewer_pid=%d\n" % viewer_pid) - handle.write("viewer_start=%s\n" % viewer_start) + handle.write("viewer_start_utc=%s\n" % viewer_start) os.rename(temporary, path) diff --git a/bin/fm-herdr-lab.sh b/bin/fm-herdr-lab.sh index 12a041f2acc..7836d999767 100755 --- a/bin/fm-herdr-lab.sh +++ b/bin/fm-herdr-lab.sh @@ -37,6 +37,9 @@ # absent; teardown refuses when that stop cannot be confirmed. set -u +# shellcheck source=bin/fm-pid-identity-lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/fm-pid-identity-lib.sh" + fm_herdr_lab_error() { echo "fm-herdr-lab: $*" >&2 } @@ -207,8 +210,10 @@ fm_herdr_lab_viewer_reason() { # printf '%s' "$out" | jq -r '.result.reason // empty' 2>/dev/null } +# Read in UTC, as bin/fm-herdr-lab-viewer.py records it, so a host time-zone +# change cannot disown a live viewer (bin/fm-pid-identity-lib.sh). fm_herdr_lab_process_start() { # - LC_ALL=C ps -p "$1" -o lstart= 2>/dev/null | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' + LC_ALL=C TZ=UTC0 ps -p "$1" -o lstart= 2>/dev/null | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' } fm_herdr_lab_process_parent() { # @@ -224,19 +229,29 @@ fm_herdr_lab_viewer_recorded_value() { # printf '%s' "$value" } +# True while still has the start time recorded under . A _utc +# record must match exactly; a bare record is a pre-upgrade local-time +# rendering, checked by fm_pid_identity_legacy_matches. +fm_herdr_lab_viewer_start_matches() { # + local recorded current + current=$(fm_herdr_lab_process_start "$3") && [ -n "$current" ] || return 1 + if recorded=$(fm_herdr_lab_viewer_recorded_value "$1" "$2_utc"); then + [ "$current" = "$recorded" ] + else + recorded=$(fm_herdr_lab_viewer_recorded_value "$1" "$2") || return 1 + fm_pid_identity_legacy_matches "$recorded" "$current" + fi +} + fm_herdr_lab_viewer_owned_pair() { # - local launcher_pid viewer_pid launcher_start viewer_start current_start parent_pid + local launcher_pid viewer_pid parent_pid launcher_pid=$(fm_herdr_lab_viewer_recorded_value "$1" launcher_pid) || return 1 viewer_pid=$(fm_herdr_lab_viewer_recorded_value "$1" viewer_pid) || return 1 case "$launcher_pid:$viewer_pid" in *[!0-9:]*) return 1 ;; esac - launcher_start=$(fm_herdr_lab_viewer_recorded_value "$1" launcher_start) || return 1 - viewer_start=$(fm_herdr_lab_viewer_recorded_value "$1" viewer_start) || return 1 - current_start=$(fm_herdr_lab_process_start "$launcher_pid") || return 1 - [ -n "$current_start" ] && [ "$current_start" = "$launcher_start" ] || return 1 - current_start=$(fm_herdr_lab_process_start "$viewer_pid") || return 1 - [ -n "$current_start" ] && [ "$current_start" = "$viewer_start" ] || return 1 + fm_herdr_lab_viewer_start_matches "$1" launcher_start "$launcher_pid" || return 1 + fm_herdr_lab_viewer_start_matches "$1" viewer_start "$viewer_pid" || return 1 parent_pid=$(fm_herdr_lab_process_parent "$viewer_pid") || return 1 [ "$parent_pid" = "$launcher_pid" ] || return 1 printf '%s %s' "$launcher_pid" "$viewer_pid" diff --git a/bin/fm-live-lab.sh b/bin/fm-live-lab.sh index 65462181a02..2d566a04dbb 100755 --- a/bin/fm-live-lab.sh +++ b/bin/fm-live-lab.sh @@ -111,6 +111,8 @@ set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" BUILDER_ROOT="$(cd "$SCRIPT_DIR/.." && pwd -P)" LAB_HOME_HELPER="$SCRIPT_DIR/fm-lab-home.sh" +# shellcheck source=bin/fm-pid-identity-lib.sh +. "$SCRIPT_DIR/fm-pid-identity-lib.sh" CLAUDE_TRUST="$SCRIPT_DIR/fm-claude-trust.sh" RECORD_NAME=.fm-live-lab RECORD_TOKEN='fm-live-lab v1' @@ -598,32 +600,47 @@ cmd_up() { # ---- down ------------------------------------------------------------------- +# Every lstart read below pins TZ=UTC0, so a host time-zone change between up +# and down cannot disown a live lab process (bin/fm-pid-identity-lib.sh). record_launch_pid() { local start case "${1:-}" in ''|*[!0-9]*) die "cannot record lab process: missing or invalid PID '${1:-}'" ;; esac - start=$(ps -o lstart= -p "$1" | awk '{$1=$1; print}') + start=$(TZ=UTC0 ps -o lstart= -p "$1" | awk '{$1=$1; print}') [ -n "$start" ] || die "cannot record start time for lab process $1" - printf 'launch_pid=%s\nlaunch_start=%s\n' "$1" "$start" >> "$ROOT/$RECORD_NAME" + printf 'launch_pid=%s\nlaunch_start_utc=%s\n' "$1" "$start" >> "$ROOT/$RECORD_NAME" +} + +# Print the recorded launch PIDs whose start time in (pid= ppid= +# lstart= rows read under TZ=UTC0) still matches. A launch_start_utc record +# must match exactly; a bare launch_start record is a pre-upgrade local-time +# rendering, checked by fm_pid_identity_legacy_matches. +lab_roots() { # + local line root="" start + while IFS= read -r line; do + case "$line" in + launch_pid=*) root=${line#launch_pid=}; case "$root" in ''|*[!0-9]*) root="" ;; esac ;; + launch_start_utc=*|launch_start=*) + [ -n "$root" ] || continue + start=$(printf '%s\n' "$1" | awk -v pid="$root" '$1 == pid { print $3, $4, $5, $6, $7 }') + case "$line" in + launch_start_utc=*) [ -n "$start" ] && [ "${line#launch_start_utc=}" = "$start" ] && echo "$root" ;; + *) fm_pid_identity_legacy_matches "${line#launch_start=}" "$start" && echo "$root" ;; + esac + root="" ;; + esac + done < "$ROOT/$RECORD_NAME" } # Resolve recorded roots only while their start times match, before tmux -# reparents their descendants. +# reparents their descendants. The root check and the tree walk read one +# snapshot, so a root PID reused between them cannot adopt a new tree. lab_pids() { - ps -axo pid=,ppid=,lstart= | awk -v record="$ROOT/$RECORD_NAME" ' - BEGIN { - while ((getline line < record) > 0) { - if (line ~ /^launch_pid=[0-9]+$/) { sub(/^launch_pid=/, "", line); root=line } - else if (line ~ /^launch_start=/ && root != "") { - sub(/^launch_start=/, "", line); starts[root]=line; root="" - } - } - close(record) - } - { - pid[NR]=$1; ppid[$1]=$2 - start=$3 " " $4 " " $5 " " $6 " " $7 - if ($1 in starts && start == starts[$1]) roots[$1]=1 - } + local snapshot roots + snapshot=$(TZ=UTC0 ps -axo pid=,ppid=,lstart=) + roots=$(lab_roots "$snapshot" | tr '\n' ' ') + printf '%s\n' "$snapshot" | awk -v matched="$roots" ' + BEGIN { split(matched, ids, /[[:space:]]+/); for (i in ids) if (ids[i] != "") roots[ids[i]]=1 } + { pid[NR]=$1; ppid[$1]=$2 } END { for (i = 1; i <= NR; i++) { p = pid[i] @@ -674,7 +691,7 @@ prune_groups() { refresh_pairs() { local snapshot - snapshot=$(ps -axo pid=,ppid=,pgid=,stat=,lstart=) + snapshot=$(TZ=UTC0 ps -axo pid=,ppid=,pgid=,stat=,lstart=) pairs=$(expand_pairs "$pairs" "$groups" "$snapshot") groups=$(prune_groups "$groups" "$pairs" "$snapshot") } @@ -685,7 +702,7 @@ live_pids() { local pid start current while IFS=$'\t' read -r pid start; do [ -n "$pid" ] || continue - current=$(ps -o stat=,lstart= -p "$pid" 2>/dev/null | awk '{$1=$1; print}') + current=$(TZ=UTC0 ps -o stat=,lstart= -p "$pid" 2>/dev/null | awk '{$1=$1; print}') case "$current" in ''|Z*) ;; *) [ "${current#* }" = "$start" ] && echo "$pid" ;; @@ -750,10 +767,10 @@ cmd_down() { own_group=$(ps -o pgid= -p "$$" | awk '{$1=$1; print}') caller_group=$(ps -o pgid= -p "$PPID" | awk '{$1=$1; print}') for pid in $pids; do - start=$(ps -o lstart= -p "$pid" 2>/dev/null | awk '{$1=$1; print}') + start=$(TZ=UTC0 ps -o lstart= -p "$pid" 2>/dev/null | awk '{$1=$1; print}') [ -n "$start" ] || continue pairs+="$pid"$'\t'"$start"$'\n' - pgid=$(ps -o pgid=,lstart= -p "$pid" 2>/dev/null | awk -v start="$start" '{ if ($2 " " $3 " " $4 " " $5 " " $6 == start) print $1 }') + pgid=$(TZ=UTC0 ps -o pgid=,lstart= -p "$pid" 2>/dev/null | awk -v start="$start" '{ if ($2 " " $3 " " $4 " " $5 " " $6 == start) print $1 }') case "$pgid" in ''|0|1|*[!0-9]*) continue ;; esac [ "$pgid" = "$own_group" ] || [ "$pgid" = "$caller_group" ] || groups+="$pgid " done diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 5b5fa96b697..8d54460946b 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -122,6 +122,8 @@ _FM_PENDING_REPLY_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/n . "$_FM_PENDING_REPLY_LIB_DIR/fm-marker-lib.sh" # shellcheck source=bin/fm-backend.sh . "$_FM_PENDING_REPLY_LIB_DIR/fm-backend.sh" +# shellcheck source=bin/fm-pid-identity-lib.sh +. "$_FM_PENDING_REPLY_LIB_DIR/fm-pid-identity-lib.sh" # Deliberately undirected: this library consumes no symbols from # bin/fm-tmux-lib.sh, so following it under ShellCheck's external-source # traversal would expand that graph for zero cross-file checks. @@ -974,7 +976,7 @@ fm_pending_reply_send_recovery() { # parent_home=$(fm_pending_reply_get "$rec" parent_home) msg=$(fm_pending_reply_recovery_message "$rec") sender_pid=${BASHPID:-$$} - sender_identity=$(fm_pending_reply_pid_identity "$sender_pid") || return 1 + sender_identity=$(fm_pid_identity "$sender_pid") || return 1 # One fresh, uncached read immediately before firing, under the same # per-correlation lock that records the send: a correlated report resolved # in between can then never be overwritten by the repost. Lock globals are @@ -1020,21 +1022,12 @@ fm_pending_reply_send_recovery() { # return 1 } -fm_pending_reply_pid_identity() { # - local pid=$1 identity - case "$pid" in ''|*[!0-9]*) return 1 ;; esac - identity=$(COLUMNS=10000 LC_ALL=C ps -p "$pid" -o lstart= -o command= 2>/dev/null) || return 1 - [ -n "$identity" ] || return 1 - printf '%s' "$identity" -} - fm_pending_reply_sender_alive() { # - local rec=$1 pid expected actual + local rec=$1 pid expected pid=$(fm_pending_reply_get "$rec" recovery_sender_pid) expected=$(fm_pending_reply_get "$rec" recovery_sender_identity) [ -n "$expected" ] || return 1 - actual=$(fm_pending_reply_pid_identity "$pid") || return 1 - [ "$actual" = "$expected" ] + fm_pid_identity_matches "$pid" "$expected" } fm_pending_reply_finish_recovery() { # diff --git a/bin/fm-pid-identity-lib.sh b/bin/fm-pid-identity-lib.sh new file mode 100644 index 00000000000..a1e85ef9a0c --- /dev/null +++ b/bin/fm-pid-identity-lib.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# fm-pid-identity-lib.sh - process identities that no time-zone change moves, +# with no source-time side effects, so a library without the wake library's +# state setup (the remote job worker) can load the same definition. +# +# An identity proves that a recorded pid still names the process that recorded +# it, across PID reuse: the process start time plus its full command line. +# The Linux-compatible /proc form uses stat field 22 (starttime, clock ticks +# since boot), which no clock step or zone change moves. The portable ps form +# reads lstart, which ps renders in the local time zone, so it is read under +# TZ=UTC0 and keyed "lstart-utc=". Builds before that pin recorded the bare +# local-time rendering; fm_pid_identity_matches is the single owner of +# comparing a recorded identity with a live pid, including that legacy form. +# +# bin/fm-extension.mjs and bin/fm-pid-identity.mjs mirror the keyed form and +# the legacy rule for the extension host, which re-reads shell-recorded claims. +# Other ps lstart readers that keep their own record shape (the remote job +# worker, teardown, and the labs) pin the same TZ=UTC0; the remote job worker +# and the labs also read their pre-pin records through +# fm_pid_identity_legacy_matches. + +# Print 's lstart and command line as ps renders them in UTC under the C +# locale, with leading blanks removed. LC_ALL=C keeps the date format +# locale-invariant, and the wide COLUMNS keeps a narrow-terminal caller from +# cutting the command short (issue #799). +_fm_pid_ps_utc() { # + local pid=$1 out + case "$pid" in + ''|*[!0-9]*) return 1 ;; + esac + out=$(COLUMNS=10000 LC_ALL=C TZ=UTC0 ps -p "$pid" -o lstart= -o command= 2>/dev/null) || return 1 + out=${out#"${out%%[![:space:]]*}"} + [ -n "$out" ] || return 1 + printf '%s\n' "$out" +} + +fm_pid_identity() { # + local pid=$1 out proc_root stat_line starttime cmdline_hex identity_key + local -a stat_fields + case "$pid" in + ''|*[!0-9]*) return 1 ;; + esac + proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} + # Prefer a Linux-compatible /proc when present: its starttime is also immune + # to the wall-clock steps that re-render the ps lstart date (observed as WSL2 + # btime drift), and the full NUL-separated cmdline keeps PID reuse a mismatch + # even on a tick collision. Git Bash/MSYS exposes these compatible files but + # its Cygwin ps rejects the portable -o fields, so capability detection must + # not key on uname. + if [ -r "$proc_root/$pid/stat" ] && [ -r "$proc_root/$pid/cmdline" ]; then + stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 + # After the final comm delimiter, array index 19 is proc stat field 22. + read -r -a stat_fields <<< "${stat_line##*)}" + [ "${#stat_fields[@]}" -ge 20 ] || return 1 + starttime=${stat_fields[19]} + case "$starttime" in + ''|*[!0-9]*) return 1 ;; + esac + cmdline_hex=$(od -An -v -tx1 "$proc_root/$pid/cmdline" 2>/dev/null | tr -d '[:space:]') || return 1 + [ -n "$cmdline_hex" ] || return 1 + [ -n "${_FM_UNAME:-}" ] || _FM_UNAME=$(uname 2>/dev/null || echo unknown) + identity_key=proc-starttime + [ "$_FM_UNAME" != Linux ] || identity_key=linux-starttime + printf '%s=%s cmdline-hex=%s\n' "$identity_key" "$starttime" "$cmdline_hex" + return 0 + fi + out=$(_fm_pid_ps_utc "$pid") || return 1 + printf 'lstart-utc=%s\n' "$out" +} + +# Parse the lstart date ("Fri Oct 2 15:14:48 2026", C locale) that begins +# . Sets _FM_PID_LSTART_SECONDS to the seconds it names when read as UTC, +# and _FM_PID_LSTART_REST to the text after it with tabs, carriage returns, and +# newlines flattened to spaces and outer blanks trimmed. Fails when does +# not begin with such a date. +_FM_PID_LSTART_SECONDS= +_FM_PID_LSTART_REST= +_fm_pid_lstart_parse() { # + local re='^[[:space:]]*[A-Z][a-z][a-z] ([A-Z][a-z][a-z]) +([0-9]{1,2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})(.*)$' + local month year day era yoe doy rest + [[ $1 =~ $re ]] || return 1 + case "${BASH_REMATCH[1]}" in + Jan) month=1 ;; Feb) month=2 ;; Mar) month=3 ;; Apr) month=4 ;; + May) month=5 ;; Jun) month=6 ;; Jul) month=7 ;; Aug) month=8 ;; + Sep) month=9 ;; Oct) month=10 ;; Nov) month=11 ;; Dec) month=12 ;; + *) return 1 ;; + esac + year=$((10#${BASH_REMATCH[6]})) + day=$((10#${BASH_REMATCH[2]})) + # Days from 1970-01-01 to the civil date (Hinnant's days_from_civil). + [ "$month" -gt 2 ] || year=$((year - 1)) + era=$((year / 400)) + yoe=$((year - era * 400)) + if [ "$month" -gt 2 ]; then + doy=$(((153 * (month - 3) + 2) / 5 + day - 1)) + else + doy=$(((153 * (month + 9) + 2) / 5 + day - 1)) + fi + _FM_PID_LSTART_SECONDS=$((((era * 146097 + yoe * 365 + yoe / 4 - yoe / 100 + doy) - 719468) * 86400 \ + + 10#${BASH_REMATCH[3]} * 3600 + 10#${BASH_REMATCH[4]} * 60 + 10#${BASH_REMATCH[5]})) + rest=${BASH_REMATCH[7]//[$'\t\r\n']/ } + rest=${rest#"${rest%%[! ]*}"} + _FM_PID_LSTART_REST=${rest%"${rest##*[! ]}"} +} + +# fm_pid_identity_legacy_matches +# True when is the unkeyed local-time form a build before the UTC +# pin wrote, and it names the same process as , the same ps fields +# rendered in UTC. The text after the date must be identical once blanks are +# flattened, and the two dates must differ by a whole quarter hour between +# -12:00 and +14:00, the range every civil zone offset falls in. So a live +# owner recorded before an upgrade still matches after any zone change, while +# PID reuse still mismatches unless an identical command line restarts on that +# exact pid at a quarter-hour-aligned second. Keyed records never reach this +# rule, so it retires with the last owner a pre-upgrade build recorded. +fm_pid_identity_legacy_matches() { # + local recorded_seconds recorded_rest offset + _fm_pid_lstart_parse "$1" || return 1 + recorded_seconds=$_FM_PID_LSTART_SECONDS + recorded_rest=$_FM_PID_LSTART_REST + _fm_pid_lstart_parse "$2" || return 1 + [ "$recorded_rest" = "$_FM_PID_LSTART_REST" ] || return 1 + offset=$((recorded_seconds - _FM_PID_LSTART_SECONDS)) + [ $((offset % 900)) -eq 0 ] || return 1 + [ "$offset" -ge -43200 ] && [ "$offset" -le 50400 ] +} + +# fm_pid_identity_matches +# 0 when is still the process names, 1 when it is not (or +# is empty), 2 when 's identity cannot be read now. A keyed +# record must match exactly; an unkeyed legacy record is checked against the +# UTC rendering by fm_pid_identity_legacy_matches. +fm_pid_identity_matches() { # + local pid=$1 recorded=$2 current utc + current=$(fm_pid_identity "$pid" 2>/dev/null) && [ -n "$current" ] || return 2 + [ -n "$recorded" ] || return 1 + [ "$current" = "$recorded" ] && return 0 + case "$current" in + lstart-utc=*) utc=${current#lstart-utc=} ;; + *) + # A /proc host compares a legacy ps record (an older pending-reply + # sender) against the ps rendering, read only for that record shape. + _fm_pid_lstart_parse "$recorded" || return 1 + utc=$(_fm_pid_ps_utc "$pid") || return 2 + ;; + esac + fm_pid_identity_legacy_matches "$recorded" "$utc" && return 0 + return 1 +} diff --git a/bin/fm-pid-identity.mjs b/bin/fm-pid-identity.mjs new file mode 100644 index 00000000000..20c33cd5d28 --- /dev/null +++ b/bin/fm-pid-identity.mjs @@ -0,0 +1,32 @@ +// fm-pid-identity.mjs - the Node mirror of the legacy-identity rule that +// bin/fm-pid-identity-lib.sh owns (fm_pid_identity_legacy_matches), for the +// extension host. Pure functions with no side effects, so a test can check +// that both implementations reach the same verdict. + +const LSTART_RE = /^\s*[A-Z][a-z]{2} ([A-Z][a-z]{2}) +([0-9]{1,2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})([\s\S]*)$/u; +const LSTART_MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + +// The seconds the leading C-locale lstart date of text names when read as +// UTC, and the text after it with blanks flattened and trimmed; null when text +// does not begin with such a date. +export function lstartParts(text) { + const match = LSTART_RE.exec(text); + const month = match ? LSTART_MONTHS.indexOf(match[1]) : -1; + if (month < 0) return null; + return { + seconds: Date.UTC(Number(match[6]), month, Number(match[2]), Number(match[3]), Number(match[4]), Number(match[5])) / 1000, + rest: match[7].replace(/[\t\r\n]/gu, " ").replace(/^ +| +$/gu, ""), + }; +} + +// True when recorded is the unkeyed local-time identity a build before the +// UTC pin wrote and names the same process as utcText, the same ps fields +// rendered in UTC: identical text after the date, and dates a whole quarter +// hour apart within the -12:00..+14:00 range of civil zone offsets. +export function legacyIdentityMatches(recorded, utcText) { + const legacy = lstartParts(recorded); + const current = lstartParts(utcText); + if (!legacy || !current || legacy.rest !== current.rest) return false; + const offset = legacy.seconds - current.seconds; + return offset % 900 === 0 && offset >= -43200 && offset <= 50400; +} diff --git a/bin/fm-procevent-lib.sh b/bin/fm-procevent-lib.sh index dcc9fd592bc..5ee1300827e 100644 --- a/bin/fm-procevent-lib.sh +++ b/bin/fm-procevent-lib.sh @@ -721,15 +721,13 @@ fm_procevent_group_alive() { # leaderless group created after PID/PGID reuse, so cleanup preserves the claim # without signalling the group or starting a replacement. fm_procevent_pid_state() { - local pid=$1 expected=$2 actual + local pid=$1 expected=$2 match=0 if ! fm_pid_alive "$pid"; then fm_procevent_group_alive "$pid" && return 3 return 1 fi - if actual=$(fm_pid_identity "$pid" 2>/dev/null); then - [ "$actual" = "$expected" ] && return 0 - return 1 - fi + fm_pid_identity_matches "$pid" "$expected" || match=$? + [ "$match" -eq 2 ] || return "$match" fm_pid_alive "$pid" || { fm_procevent_group_alive "$pid" && return 3; return 1; } return 2 } diff --git a/bin/fm-remote-job-lib.sh b/bin/fm-remote-job-lib.sh index ce6e4090737..31d0f77e9c2 100755 --- a/bin/fm-remote-job-lib.sh +++ b/bin/fm-remote-job-lib.sh @@ -95,6 +95,11 @@ # bin/fm-remote-job-reap-orphans.sh uses it to reap workers that were already # orphaned that way. +_FM_REMOTE_JOB_LIB_DIR=${BASH_SOURCE[0]%/*} +[ "$_FM_REMOTE_JOB_LIB_DIR" != "${BASH_SOURCE[0]}" ] || _FM_REMOTE_JOB_LIB_DIR=. +# shellcheck source=bin/fm-pid-identity-lib.sh +. "$_FM_REMOTE_JOB_LIB_DIR/fm-pid-identity-lib.sh" + FM_REMOTE_JOB_LABEL=dev.firstmate.remote-job FM_REMOTE_JOB_MAX_BYTES=${FM_REMOTE_JOB_MAX_BYTES:-1048576} FM_REMOTE_JOB_QUEUE_TIMEOUT=${FM_REMOTE_JOB_QUEUE_TIMEOUT:-360} @@ -812,7 +817,7 @@ fm_remote_job_stage_owner_alive() { # [ "$pid" -gt 1 ] || return 1 recorded_start=$(fm_remote_job_read_single_line "$stage/.owner-start" 256 2>/dev/null) || return 1 actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1 - [ "$recorded_start" = "$actual_start" ] + fm_remote_job_start_matches "$recorded_start" "$actual_start" } fm_remote_job_reap_stale() { # @@ -946,13 +951,31 @@ fm_remote_job_worker_ready_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.r fm_remote_job_worker_identity_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.identity"; } fm_remote_job_worker_lock_path() { printf '%s\n' "$FM_REMOTE_JOB_STATE/worker.lock"; } +# A process start record: lstart read in UTC under the C locale and keyed like +# fm_pid_identity's ps form (bin/fm-pid-identity-lib.sh), so no host time-zone +# change makes a live owner look dead. Compare records with +# fm_remote_job_start_matches, which also reads the local-time records builds +# before the UTC pin wrote. fm_remote_job_process_start() { local pid=$1 ps_bin value if [ -x /bin/ps ]; then ps_bin=/bin/ps; elif [ -x /usr/bin/ps ]; then ps_bin=/usr/bin/ps; else return 1; fi - value=$("$ps_bin" -p "$pid" -o lstart= 2>/dev/null) || return 1 + value=$(LC_ALL=C TZ=UTC0 "$ps_bin" -p "$pid" -o lstart= 2>/dev/null) || return 1 + value=${value#"${value%%[![:space:]]*}"} + value=${value%"${value##*[![:space:]]}"} [ -n "$value" ] || return 1 case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac - printf '%s\n' "$value" + printf 'lstart-utc=%s\n' "$value" +} + +# True when , a fresh fm_remote_job_process_start, names the process +# the non-empty start record was written for. +fm_remote_job_start_matches() { # + [ -n "$1" ] || return 1 + [ "$1" = "$2" ] && return 0 + case "$2" in + lstart-utc=*) fm_pid_identity_legacy_matches "$1" "${2#lstart-utc=}" ;; + *) return 1 ;; + esac } fm_remote_job_process_command() { @@ -1056,7 +1079,7 @@ fm_remote_job_lock_owner_matches_process() { [ "$pid" -gt 1 ] || return 1 recorded_start=$(fm_remote_job_read_single_line "$lock/start" 256) || return 1 actual_start=$(fm_remote_job_process_start "$pid") || return 1 - [ "$recorded_start" = "$actual_start" ] || return 1 + fm_remote_job_start_matches "$recorded_start" "$actual_start" || return 1 recorded_command=$(fm_remote_job_read_single_line "$lock/command" 8192) || return 1 actual_command=$(fm_remote_job_process_command "$pid") || return 1 [ "$recorded_command" = "$actual_command" ] || return 1 @@ -1093,12 +1116,16 @@ fm_remote_job_code_identity() { # local root=$1 account_home=$2 git_bin root_hash library_hash worker_hash root=$(fm_remote_job_canonical_existing_dir "$root") || return 1 [ -f "$root/bin/fm-remote-job-lib.sh" ] && [ ! -L "$root/bin/fm-remote-job-lib.sh" ] || return 1 + [ -f "$root/bin/fm-pid-identity-lib.sh" ] && [ ! -L "$root/bin/fm-pid-identity-lib.sh" ] || return 1 [ -f "$root/bin/fm-remote-job-worker.sh" ] && [ ! -L "$root/bin/fm-remote-job-worker.sh" ] || return 1 fm_remote_job_compose_operator_path "$account_home" >/dev/null git_bin=$(fm_remote_job_operator_tool git 2>/dev/null || true) [ -n "$git_bin" ] || return 1 root_hash=$(printf '%s' "$root" | "$git_bin" hash-object --stdin 2>/dev/null) || return 1 - library_hash=$("$git_bin" hash-object -- "$root/bin/fm-remote-job-lib.sh" 2>/dev/null) || return 1 + # The library hash covers the identity library it sources, so a change to + # either replaces a worker still running the old definitions. + library_hash=$(cat -- "$root/bin/fm-remote-job-lib.sh" "$root/bin/fm-pid-identity-lib.sh" 2>/dev/null \ + | "$git_bin" hash-object --stdin 2>/dev/null) || return 1 worker_hash=$("$git_bin" hash-object -- "$root/bin/fm-remote-job-worker.sh" 2>/dev/null) || return 1 case "$root_hash:$library_hash:$worker_hash" in *[!0-9a-f:]*) return 1 ;; esac [ -n "$root_hash" ] && [ -n "$library_hash" ] && [ -n "$worker_hash" ] || return 1 diff --git a/bin/fm-remote-job-worker.sh b/bin/fm-remote-job-worker.sh index 118d75c3493..5357c485729 100755 --- a/bin/fm-remote-job-worker.sh +++ b/bin/fm-remote-job-worker.sh @@ -338,7 +338,7 @@ worker_supervisor_identity_status() { # worker_process_or_group_alive process "$pid" && return 2 return 1 } - [ "$recorded_start" = "$actual_start" ] && return 0 + fm_remote_job_start_matches "$recorded_start" "$actual_start" && return 0 return 1 } @@ -356,7 +356,7 @@ worker_group_identity_status() { # worker_process_or_group_alive group "$pid" && return 0 return 1 } - [ "$recorded_start" = "$actual_start" ] && return 0 + fm_remote_job_start_matches "$recorded_start" "$actual_start" && return 0 return 1 } @@ -448,7 +448,7 @@ worker_lane_identity_matches() { # local pid=$1 start=$2 actual_start [ -n "$start" ] || return 1 actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1 - [ "$actual_start" = "$start" ] + fm_remote_job_start_matches "$start" "$actual_start" } worker_stop_active_execution() { @@ -573,7 +573,7 @@ worker_claim_owner_alive() { # if [ -e "$claim/owner_start" ] || [ -L "$claim/owner_start" ]; then recorded_start=$(fm_remote_job_read_single_line "$claim/owner_start" 256 2>/dev/null) || return 1 actual_start=$(fm_remote_job_process_start "$pid" 2>/dev/null) || return 1 - [ "$recorded_start" = "$actual_start" ] + fm_remote_job_start_matches "$recorded_start" "$actual_start" return fi kill -0 "$pid" 2>/dev/null diff --git a/bin/fm-supervise-daemon.sh b/bin/fm-supervise-daemon.sh index a2a7664f4fb..7abf6437fc0 100755 --- a/bin/fm-supervise-daemon.sh +++ b/bin/fm-supervise-daemon.sh @@ -14,14 +14,16 @@ # or omits anything it names a state/.subsuper-digests/ file holding every # buffered event verbatim. # -# PRESENCE-GATING (the /afk contract). The daemon is the away-mode engine: it -# injects ONLY when the durable away-mode flag state/.afk is present. Invoking -# the /afk skill sets that flag and starts this daemon; any real (unmarked) -# user message clears it and firstmate resumes full responsiveness. -# When afk is off, normal fm-watch.sh always-on triage is the active mechanism. -# Any buffered daemon escalations that remain while afk is off survive in -# state/.subsuper-escalations and are flushed on the next "while you were out" -# catch-up or when afk is re-entered. +# PRESENCE-GATING (the /afk contract). The daemon is the away-mode engine and +# lives only while the durable away-mode flag state/.afk is present: it injects +# only then. Invoking the /afk skill sets that flag and starts this daemon; any +# real (unmarked) user message clears it and firstmate resumes full +# responsiveness. Once the flag is gone, or once this daemon no longer holds its +# singleton lock, it exits on its own before it drains another wake, so the +# attended firstmate's own watcher receives every wake again; a daemon left +# running would drain and acknowledge each one into a buffer nothing delivers. +# Escalations it could not deliver survive in state/.subsuper-escalations as +# evidence for the return catch-up. # # IN-BAND OPERATIONAL INPUT. bin/fm-operational-input.sh constructs every # current daemon injection as the typed away-supervisor kind after the stable @@ -202,6 +204,9 @@ FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" # shellcheck source=bin/fm-busy-lib.sh . "$FM_DAEMON_DIR/fm-busy-lib.sh" +# shellcheck source=bin/fm-afk-daemon-lib.sh +. "$FM_DAEMON_DIR/fm-afk-daemon-lib.sh" + # --- tunables --------------------------------------------------------------- # Supervisor backends this daemon knows how to inject into today. zellij, orca, # and cmux are real backends elsewhere in firstmate (bin/fm-backend.sh) but this @@ -1420,8 +1425,9 @@ inject_msg() { # [state] local msg=$1 state target backend retries sleep_s verdict composer encoded bytes errf err='' body state="${2:-$(_state_root)}" # (1) Presence-gate: inject ONLY when afk is active. When afk is off, the - # daemon self-handles and stays quiet; firstmate drives the normal always-on - # watcher triage. Escalations buffer and survive for the next catch-up flush. + # daemon is about to exit (PRESENCE-GATING above) and firstmate drives the + # normal always-on watcher triage; escalations stay buffered for the return + # catch-up. INJECT_LAST_FAILURE= INJECT_SUBMIT_ATTEMPTED=0 afk_active "$state" || { INJECT_LAST_FAILURE="deferred: afk inactive"; log "inject $INJECT_LAST_FAILURE"; return 1; } @@ -1775,6 +1781,7 @@ fm_super_main() { fi exit 1 fi + local LOCK_OWNER=${FM_LOCK_OWNER_DIR:-} echo "$$" > "$PIDFILE" # The recorded identity is what proves this daemon still owns supervision after # its watcher child exits (fm_afk_daemon_owns_supervision, read by the turn-end @@ -1786,6 +1793,30 @@ fm_super_main() { log "warn: could not record this daemon's process identity; the turn-end guard cannot recognize away-mode supervision" fi + # Release only what this instance owns: the lock and pid file may already + # name a successor daemon. + release_instance() { + fm_lock_release "$LOCK" 2>/dev/null || true + [ "$(cat "$PIDFILE" 2>/dev/null || true)" != "$$" ] || rm -f "$PIDFILE" 2>/dev/null || true + } + + # The daemon exists only for away or quiet mode (state/.afk) and only while it + # holds the singleton lock. Without either it would keep draining and + # acknowledging every wake into a buffer that nothing delivers, so it exits. + holds_lock() { + [ "$(cat "$LOCK/pid" 2>/dev/null || true)" = "$$" ] || return 1 + [ -z "$LOCK_OWNER" ] || fm_lock_points_to_owner "$LOCK" "$LOCK_OWNER" + } + exit_reason() { + if ! afk_active "$STATE"; then + printf 'away and quiet mode are no longer active' + elif ! holds_lock; then + printf 'this daemon no longer holds %s' "$LOCK" + else + return 1 + fi + } + # --- auto-discover the supervisor BACKEND (tmux vs herdr) first ----------- # Priority: FM_SUPERVISOR_BACKEND override > $TMUX_PANE (tmux) > $HERDR_ENV=1 # (herdr) > tmux fallback. Resolved before the target below, since target @@ -1817,8 +1848,7 @@ fm_super_main() { if ! fm_backend_list_contains "$FM_SUPERVISOR_SUPPORTED_BACKENDS" "$BACKEND"; then echo "error: away-mode daemon does not support supervisor backend '$BACKEND' yet (supported: $FM_SUPERVISOR_SUPPORTED_BACKENDS); set FM_SUPERVISOR_BACKEND=tmux|herdr and FM_SUPERVISOR_TARGET to run firstmate's own pane under a supported backend" >&2 log "startup failed: unsupported supervisor backend '$BACKEND' (source=$backend_source)" - fm_lock_release "$LOCK" 2>/dev/null || true - rm -f "$PIDFILE" 2>/dev/null || true + release_instance exit 1 fi @@ -1855,8 +1885,7 @@ fm_super_main() { if ! fm_backend_target_exists "$BACKEND" "$TARGET"; then echo "error: supervisor target '$TARGET' does not resolve to a $BACKEND pane; set FM_SUPERVISOR_TARGET" >&2 log "startup failed: target '$TARGET' not found (backend=$BACKEND)" - fm_lock_release "$LOCK" 2>/dev/null || true - rm -f "$PIDFILE" 2>/dev/null || true + release_instance exit 1 fi @@ -1878,8 +1907,7 @@ fm_super_main() { if [ -n "${CUR_TMP:-}" ]; then rm -f "$CUR_TMP" 2>/dev/null || true fi - fm_lock_release "$LOCK" 2>/dev/null || true - rm -f "$PIDFILE" 2>/dev/null || true + release_instance log "daemon shutting down" exit 0 } @@ -1911,8 +1939,16 @@ fm_super_main() { WATCHER_PID=$! } - local rc reason + local rc reason why while true; do + # --- presence-gated exit ------------------------------------------------- + # Checked before anything below can drain or acknowledge a wake, so the + # attended firstmate receives every wake once away mode has ended. + if why=$(exit_reason); then + log "exiting: $why" + cleanup + fi + # --- pane-gone guard (preserved) --------------------------------------- # With the #29 watcher's enqueue-before-suppress, a wake is no longer # swallowed by running the watcher with no injection target. We still back @@ -1957,6 +1993,12 @@ fm_super_main() { sleep "${HOUSEKEEPING_TICK:-$HOUSEKEEPING_TICK_DEFAULT}" continue fi + if why=$(exit_reason); then + # The child is reaped; never signal its pid again. + WATCHER_PID="" + log "exiting before handling wake '$reason' (it stays queued): $why" + cleanup + fi log "wake: $reason" if ! handle_durable_wakes "$reason" "$STATE"; then log "durable wake handling was not acknowledged; restarting for recovery" diff --git a/bin/fm-supervision-engine-lib.sh b/bin/fm-supervision-engine-lib.sh index 7bc4e1d0a30..272ee41ce64 100644 --- a/bin/fm-supervision-engine-lib.sh +++ b/bin/fm-supervision-engine-lib.sh @@ -182,7 +182,7 @@ fm_supervision_host_outcomes_drained() { # fm_supervision_host_main_key : print the key of the current main # session, which changes at every main session start: the session-lock holder, -# a checksum of its process identity (bin/fm-wake-lib.sh fm_pid_identity), and +# a checksum of its process identity (bin/fm-pid-identity-lib.sh), and # a checksum of its session sidecar, so a later session given a recycled lock # pid never shares it. The host keys its engine conversation and broken-session # latch to it; the dialog mirror (bin/fm-host-mirror.sh) keys each entry and @@ -252,8 +252,8 @@ fm_supervision_engine_bin() { printf '%s\n' "$bin" } -# Print a process's identity (bin/fm-wake-lib.sh fm_pid_identity) on one -# line, the form the descendant ledger records and compares. +# Print a process's identity (bin/fm-pid-identity-lib.sh) on one line, the +# form the descendant ledger and the host records keep. _fm_engine_identity() { # local identity identity=$(fm_pid_identity "$1" 2>/dev/null) || return 1 @@ -261,6 +261,17 @@ _fm_engine_identity() { # printf '%s\n' "$identity" | tr '\t\n' ' ' | sed 's/ *$//' } +# True while is still the process a non-empty _fm_engine_identity record +# names, including a record an older build wrote in local time +# (fm_pid_identity_matches owns that rule). +_fm_engine_identity_matches() { # + local current + [ -n "$2" ] || return 1 + current=$(_fm_engine_identity "$1") || return 1 + [ "$current" = "$2" ] && return 0 + fm_pid_identity_matches "$1" "$2" +} + # Print " " for every process. _fm_engine_process_table() { ps -A -o pid= -o ppid= 2>/dev/null @@ -306,14 +317,13 @@ _fm_engine_snapshot_descendants() { # that is still alive under its recorded identity. A recycled pid never # matches its recorded identity, so it is never signalled. _fm_engine_reap() { - local ledger=$1 pid identity current signal survivors i + local ledger=$1 pid identity signal survivors i [ -s "$ledger" ] || return 0 for signal in TERM KILL; do survivors=0 while IFS="$(printf '\t')" read -r pid identity; do fm_pid_alive "$pid" || continue - current=$(_fm_engine_identity "$pid") || continue - [ "$current" = "$identity" ] || continue + _fm_engine_identity_matches "$pid" "$identity" || continue kill "-$signal" "$pid" 2>/dev/null || true survivors=$((survivors + 1)) done < "$ledger" diff --git a/bin/fm-supervision-host.sh b/bin/fm-supervision-host.sh index 26555b69633..5c157d541e6 100755 --- a/bin/fm-supervision-host.sh +++ b/bin/fm-supervision-host.sh @@ -323,14 +323,14 @@ forget_process() { # stop_recorded() { # local pid=$1 identity=$2 limit=$(( ${3:-10} * 10 )) i fm_pid_alive "$pid" || return 0 - [ -n "$identity" ] && [ "$(identity_of "$pid")" = "$identity" ] || return 0 + _fm_engine_identity_matches "$pid" "$identity" || return 0 kill -TERM "$pid" 2>/dev/null || return 0 i=0 while [ "$i" -lt "$limit" ] && fm_pid_alive "$pid"; do sleep 0.1 i=$((i + 1)) done - if fm_pid_alive "$pid" && [ "$(identity_of "$pid")" = "$identity" ]; then + if fm_pid_alive "$pid" && _fm_engine_identity_matches "$pid" "$identity"; then kill -KILL "$pid" 2>/dev/null || true fi } @@ -383,7 +383,7 @@ activate() { if [ -f "$LEFT_RECORD" ]; then pid='' identity='' IFS="$(printf '\t')" read -r pid identity < "$LEFT_RECORD" || true - if fm_pid_alive "$pid" && [ -n "$identity" ] && [ "$(identity_of "$pid")" = "$identity" ]; then + if fm_pid_alive "$pid" && _fm_engine_identity_matches "$pid" "$identity"; then LEFT_ARM=$pid else rm -f "$LEFT_RECORD" @@ -399,7 +399,7 @@ activate() { stop_engine_turn() { local pid='' identity='' i limit [ -f "$ENGINE_PID_FILE" ] && IFS="$(printf '\t')" read -r pid identity < "$ENGINE_PID_FILE" - if [ -n "$pid" ] && fm_pid_alive "$pid" && [ "$(identity_of "$pid")" = "$identity" ]; then + if [ -n "$pid" ] && fm_pid_alive "$pid" && _fm_engine_identity_matches "$pid" "$identity"; then kill -TERM "$pid" 2>/dev/null || true fi limit=$(( (ENGINE_GRACE + 10) * 10 )) diff --git a/bin/fm-teardown.sh b/bin/fm-teardown.sh index 24ed4644c76..1fa2471ddfa 100755 --- a/bin/fm-teardown.sh +++ b/bin/fm-teardown.sh @@ -343,6 +343,7 @@ for _teardown_source in \ fm-nm-run-lib.sh \ fm-wake-lib.sh \ fm-path-lib.sh \ + fm-pid-identity-lib.sh \ fm-lease-lib.sh do teardown_require_source "$SCRIPT_DIR/$_teardown_source" @@ -2080,6 +2081,8 @@ $out EOF } +# A start-only identity, compared within one teardown run. The ps form reads +# lstart in UTC for the reason bin/fm-pid-identity-lib.sh gives. task_process_identity() { # local pid=$1 proc_root stat_line starttime value local -a stat_fields @@ -2093,7 +2096,7 @@ task_process_identity() { # printf 'starttime=%s\n' "$starttime" return 0 fi - value=$(LC_ALL=C ps -p "$pid" -o lstart= 2>/dev/null) || return 1 + value=$(LC_ALL=C TZ=UTC0 ps -p "$pid" -o lstart= 2>/dev/null) || return 1 value=$(fm_nm_trim "$value") [ -n "$value" ] || return 1 case "$value" in *$'\n'*|*$'\r'*) return 1 ;; esac diff --git a/bin/fm-wake-lib.sh b/bin/fm-wake-lib.sh index dfa387079b0..7144b2ad011 100755 --- a/bin/fm-wake-lib.sh +++ b/bin/fm-wake-lib.sh @@ -12,6 +12,8 @@ FM_WAKE_QUEUE_LOCK="${FM_WAKE_QUEUE_LOCK:-$STATE/.wake-queue.lock}" FM_LOCK_STALE_AFTER="${FM_LOCK_STALE_AFTER:-2}" # shellcheck source=bin/fm-path-lib.sh . "$FM_WAKE_LIB_DIR/fm-path-lib.sh" +# shellcheck source=bin/fm-pid-identity-lib.sh +. "$FM_WAKE_LIB_DIR/fm-pid-identity-lib.sh" # Resolved once at source time: fm_pid_identity and fm_path_mtime run inside 0.2s # confirm and 0.5s attach polls, and forking uname per call is a measurable cost on # the platform (Git Bash/MSYS) that already pays the highest fork price. @@ -65,48 +67,6 @@ fm_pid_alive() { kill -0 "$pid" 2>/dev/null } -fm_pid_identity() { - local pid=$1 out proc_root stat_line starttime cmdline_hex identity_key - local -a stat_fields - case "$pid" in - ''|*[!0-9]*) return 1 ;; - esac - proc_root=${FM_PROC_ROOT_OVERRIDE:-/proc} - # Prefer a Linux-compatible /proc when present: stat field 22 (starttime, clock ticks since boot) is - # immune to the wall-clock steps that re-render the ps lstart fallback's date - # (observed as WSL2 btime drift) and would evict a live watcher; combining the - # full NUL-separated cmdline keeps PID reuse a mismatch even on a tick collision. - # Git Bash/MSYS exposes these compatible files but its Cygwin ps rejects the - # portable fallback's -o fields, so capability detection must not key on uname. - if [ -r "$proc_root/$pid/stat" ] && [ -r "$proc_root/$pid/cmdline" ]; then - stat_line=$(cat "$proc_root/$pid/stat" 2>/dev/null) || return 1 - # After the final comm delimiter, array index 19 is proc stat field 22. - read -r -a stat_fields <<< "${stat_line##*)}" - [ "${#stat_fields[@]}" -ge 20 ] || return 1 - starttime=${stat_fields[19]} - case "$starttime" in - ''|*[!0-9]*) return 1 ;; - esac - cmdline_hex=$(od -An -v -tx1 "$proc_root/$pid/cmdline" 2>/dev/null | tr -d '[:space:]') || return 1 - [ -n "$cmdline_hex" ] || return 1 - identity_key=proc-starttime - [ "$_FM_UNAME" != Linux ] || identity_key=linux-starttime - printf '%s=%s cmdline-hex=%s\n' "$identity_key" "$starttime" "$cmdline_hex" - return 0 - fi - # Pin LC_ALL=C so lstart's date format is locale-invariant: the identity is - # written under one locale but re-read under the machine's ambient locale, which - # would otherwise mismatch on a non-C locale (e.g. ko_KR) and reject a live watcher. - # Pin COLUMNS wide so the command column is never cut to the ambient terminal - # width: the identity is written from a wide shell but re-read inside a - # narrow-COLUMNS hook, where a truncated command would likewise reject a live - # watcher (issue #799). This mirrors fm_pending_reply_pid_identity, which pins the - # same width for the same reason. - out=$(COLUMNS=10000 LC_ALL=C ps -p "$pid" -o lstart= -o command= 2>/dev/null) || return 1 - [ -n "$out" ] || return 1 - printf '%s\n' "$out" | sed 's/^[[:space:]]*//' -} - fm_path_mtime() { if [ "$_FM_UNAME" = Darwin ]; then /usr/bin/stat -f %m "$1" 2>/dev/null @@ -168,7 +128,7 @@ fm_watcher_lock_unheld() { FM_WATCHER_MATCHED_IDENTITY= fm_watcher_lock_matches_pid() { - local state=$1 watch_path=$2 pid=$3 home=${4:-$FM_HOME} lockdir lock_home lock_path lock_identity current_identity + local state=$1 watch_path=$2 pid=$3 home=${4:-$FM_HOME} lockdir lock_home lock_path lock_identity FM_WATCHER_MATCHED_IDENTITY= lockdir="$state/.watch.lock" lock_home=$(cat "$lockdir/fm-home" 2>/dev/null || true) @@ -177,8 +137,7 @@ fm_watcher_lock_matches_pid() { [ "$lock_home" = "$home" ] || return 1 [ "$lock_path" = "$watch_path" ] || return 1 [ -n "$lock_identity" ] || return 1 - current_identity=$(fm_pid_identity "$pid") || return 1 - [ "$current_identity" = "$lock_identity" ] || return 1 + fm_pid_identity_matches "$pid" "$lock_identity" || return 1 FM_WATCHER_MATCHED_IDENTITY=$lock_identity } @@ -363,16 +322,14 @@ fm_extension_pair_owns_supervision() { # /dev/null) || return 1 fm_pid_alive "$pid" || return 1 recorded=$(cat "$lockdir/pid-identity" 2>/dev/null) || return 1 [ -n "$recorded" ] || return 1 - current=$(fm_pid_identity "$pid" 2>/dev/null) || return 1 - [ -n "$current" ] || return 1 - [ "$current" = "$recorded" ] + fm_pid_identity_matches "$pid" "$recorded" } # fm_afk_mode @@ -1646,7 +1603,7 @@ fm_failure_episode_reset() { # also records "session_pid=S recovery_generation=G", binding that # handling turn to its live session-lock owner and watcher recovery episode. # Line 2 is the claiming process's pid-identity, the same identity every other -# supervision lock in this repo records (fm_pid_identity above). The +# supervision lock in this repo records (bin/fm-pid-identity-lib.sh). The # identity is MANDATORY: a claimant that cannot record it does not claim # (continuity falls to the synchronous guard), and the identity is read # from the ledger entry alone - never substituted from any lock - so a @@ -1761,7 +1718,7 @@ fm_autoarm_ledger_read() { # # build's entry gets its deference from its held role-carrying lock through # the legacy shim, and anything else must not defer. fm_autoarm_claim_open() { # [grace] - local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} epoch current + local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} epoch epoch="$state/.claude-autoarm-epoch" case "$grace" in ''|*[!0-9]*|0) grace=300 ;; @@ -1770,9 +1727,7 @@ fm_autoarm_claim_open() { # [grace] [ "$FM_AUTOARM_OUTCOME" = arming ] || return 1 fm_pid_alive "$FM_AUTOARM_OWNER" || return 1 [ -n "$FM_AUTOARM_IDENTITY" ] || return 1 - current=$(fm_pid_identity "$FM_AUTOARM_OWNER" 2>/dev/null) || return 1 - [ -n "$current" ] || return 1 - [ "$current" = "$FM_AUTOARM_IDENTITY" ] || return 1 + fm_pid_identity_matches "$FM_AUTOARM_OWNER" "$FM_AUTOARM_IDENTITY" || return 1 if [ "$(fm_path_age "$epoch")" -ge "$grace" ] \ && [ "$(fm_path_age "$state/.last-watcher-beat")" -ge "$grace" ]; then return 1 @@ -1943,7 +1898,7 @@ fm_autoarm_reset_owned() { # # and the watcher beacon are older than the guard grace (the same stuck # proof as fm_autoarm_claim_open). fm_autoarm_claim_abandoned() { # [grace] - local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} epoch lock role pid owner outcome recorded current + local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} epoch lock role pid owner outcome recorded match lock="$state/.claude-autoarm.lock" epoch="$state/.claude-autoarm-epoch" case "$grace" in @@ -1957,9 +1912,10 @@ fm_autoarm_claim_abandoned() { # [grace] ''|*[!0-9]*) return 1 ;; esac recorded=$(cat "$lock/pid-identity" 2>/dev/null || true) - if [ -n "$recorded" ] && current=$(fm_pid_identity "$pid" 2>/dev/null) \ - && [ -n "$current" ] && [ "$current" != "$recorded" ]; then - return 0 + if [ -n "$recorded" ]; then + match=0 + fm_pid_identity_matches "$pid" "$recorded" || match=$? + [ "$match" -ne 1 ] || return 0 fi owner=$(_fm_autoarm_epoch_field "$epoch" owner_pid) || return 1 [ "$owner" = "$pid" ] || return 1 @@ -1994,7 +1950,7 @@ fm_autoarm_claim_abandoned() { # [grace] # TERM and the ledger graft below, keeping the documented bounded # upgrade-window residual instead of the deadlock. fm_autoarm_release_abandoned() { # [grace] - local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} lock steal epoch lock_pid recorded current owner line1 tmp i + local state=$1 grace=${2:-${FM_GUARD_GRACE:-300}} lock steal epoch lock_pid recorded owner line1 tmp i lock="$state/.claude-autoarm.lock" steal="$lock.steal" epoch="$state/.claude-autoarm-epoch" @@ -2007,8 +1963,7 @@ fm_autoarm_release_abandoned() { # [grace] lock_pid=$(cat "$lock/pid" 2>/dev/null || true) recorded=$(cat "$lock/pid-identity" 2>/dev/null || true) if [ -n "$recorded" ] && fm_pid_alive "$lock_pid" \ - && current=$(fm_pid_identity "$lock_pid" 2>/dev/null) \ - && [ -n "$current" ] && [ "$current" = "$recorded" ]; then + && fm_pid_identity_matches "$lock_pid" "$recorded"; then # A live pid still answering to the recorded identity IS the genuine # legacy owner (proven stuck or blocked after a terminal write): retire it # before removing its lock, because old-build code cannot re-check @@ -2302,7 +2257,7 @@ fm_wake_grant_rows_valid() { # # not a match, so uncertainty reads as "no live owner". fm_wake_branch_owner_matches() { # [] [] local file=$1 expected_pid=${2:-} expected_generation=${3:-} - local version pid identity generation current extra + local version pid identity generation extra [ -f "$file" ] && [ ! -L "$file" ] || return 1 exec 8< "$file" || return 1 IFS= read -r version <&8 || { exec 8<&-; return 1; } @@ -2316,8 +2271,7 @@ fm_wake_branch_owner_matches() { # [] [] case "$generation" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac [ -z "$expected_pid" ] || [ "$pid" = "$expected_pid" ] || return 1 [ -z "$expected_generation" ] || [ "$generation" = "$expected_generation" ] || return 1 - current=$(fm_pid_identity "$pid" 2>/dev/null) || return 1 - [ -n "$current" ] && [ "$current" = "$identity" ] + fm_pid_identity_matches "$pid" "$identity" } # 0 when a branch grant is currently reserving rows: a valid row snapshot whose diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index 3a996a0320e..4307dd46d75 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -35,6 +35,9 @@ # watcher: FAILED - attached watcher pid= stalled (beacon s at or past hard bound s) # - the followed holder is alive but its beacon # reached the stall bound +# watcher: FAILED - leftover away daemon pid= did not stop, so it still owns this home's watcher +# watcher: FAILED - a leftover away daemon pid= still runs this home's watcher after a takeover +# - the takeover below could not end that daemon # It NEVER reports started/attached/healthy off a stale beacon or a dead/reused pid: a # stale-beacon or dead-pid holder either self-heals (the fresh child steals the # dead lock per the singleton self-eviction/steal path and is confirmed) or this @@ -53,6 +56,15 @@ # so the failure is loud. A live cycle already present means re-arm attaches - do # not start a second watcher. # +# The one exception is a watcher whose live parent is an away daemon while +# state/.afk is absent: that daemon outlived away mode and would take every +# wake, so the arm never attaches to it. Before any health or beacon check, in +# plain, --restart, and --take-over modes, it stops that daemon (which stops its +# own watcher), then stops this home's watcher if one remains, then re-runs +# itself in place with its own arguments to own a fresh cycle +# (take_over_from_leftover_daemon; bin/fm-afk-daemon-lib.sh proves the +# ownership). The attach paths repeat that check for a successor that appears later. +# # Every observed watcher cycle appends one tab-separated lifecycle record to # state/.watch-cycle-exits.log. The arm layer owns that bounded ledger; it records # arm/watcher identities, timestamps, exit/signal classification, beacon age, @@ -121,6 +133,8 @@ if [ "${FM_GATE_REFUSE_BYPASS:-}" != 1 ]; then fi # shellcheck source=bin/fm-wake-lib.sh . "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-afk-daemon-lib.sh +. "$SCRIPT_DIR/fm-afk-daemon-lib.sh" WATCH="$SCRIPT_DIR/fm-watch.sh" WATCH_LOCK="$STATE/.watch.lock" @@ -317,6 +331,37 @@ report_attached() { echo "watcher: attached pid=$HEALTHY_PID (beacon ${age}s)" } +# An away daemon owns triage only while state/.afk exists. A healthy watcher +# whose parent is a daemon while that flag is absent belongs to a daemon that +# outlived away mode: following it would hand every wake to that daemon, which +# drains and acknowledges it into a buffer nothing delivers. So before attaching +# to such a holder, stop that daemon (it stops its own watcher) and this home's +# watcher, then re-run this arm in place with its own arguments to own a fresh +# cycle. Only the daemon that bin/fm-afk-daemon-lib.sh proves runs this home's +# watcher is signalled. +take_over_from_leftover_daemon() { # + local holder=$1 identity=$2 daemon survivors + [ ! -e "$STATE/.afk" ] || return 0 + daemon=$(fm_afk_daemon_watcher_owner "$STATE" "$WATCH" "$FM_HOME") || return 0 + [ "$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)" = "$holder" ] || return 0 + [ "$cycle_active" -eq 1 ] || cycle_begin "$holder" leftover-daemon "$identity" + if [ "${FM_WATCH_ARM_TOOK_OVER:-}" = 1 ]; then + cycle_log_append none none leftover-daemon-returned "daemon:$daemon" + echo "watcher: FAILED - a leftover away daemon pid=$daemon still runs this home's watcher after a takeover" + exit 1 + fi + if ! survivors=$(fm_afk_daemon_stop "$daemon"); then + cycle_log_append none none leftover-daemon-survived "daemon:$survivors" + echo "watcher: FAILED - leftover away daemon pid=$daemon did not stop, so it still owns this home's watcher" + exit 1 + fi + stop_home_watcher || exit 1 + cycle_log_append none none leftover-daemon-stopped "daemon:$daemon" + [ -z "${child_out:-}" ] || rm -f "$child_out" 2>/dev/null || true + trap - HUP TERM INT + FM_WATCH_ARM_TOOK_OVER=1 exec "$SCRIPT_DIR/fm-watch-arm.sh" ${ARM_ARGS[@]+"${ARM_ARGS[@]}"} +} + # Give a successor the same bounded confirmation window used for a fresh child. # Adapter-owned continuations normally win immediately, but the bound avoids a # false failure when process-close delivery and lock publication cross briefly. @@ -397,6 +442,7 @@ attach_and_wait() { while :; do if healthy_watcher; then if [ "$HEALTHY_PID" != "$attached_pid" ] || [ "$HEALTHY_IDENTITY" != "$cycle_watcher_identity" ]; then + take_over_from_leftover_daemon "$HEALTHY_PID" "$HEALTHY_IDENTITY" cycle_log_append unknown unknown lock-replaced "attached:$HEALTHY_PID" attached_pid=$HEALTHY_PID cycle_begin "$attached_pid" attached "$HEALTHY_IDENTITY" @@ -416,6 +462,7 @@ attach_and_wait() { return 1 fi if wait_for_healthy_successor; then + take_over_from_leftover_daemon "$HEALTHY_PID" "$HEALTHY_IDENTITY" cycle_log_append unknown unknown attached-cycle-ended "attached:$HEALTHY_PID" attached_pid=$HEALTHY_PID cycle_begin "$attached_pid" attached "$HEALTHY_IDENTITY" @@ -475,6 +522,7 @@ handling_successor_generation() { esac } +ARM_ARGS=("$@") mode=arm handling_generation= handling_watcher_pid= @@ -531,6 +579,13 @@ stop_home_watcher() { fi } +# The leftover-daemon takeover (header) runs before any health or beacon +# check, so a stale beacon cannot hide that daemon's watcher. +if [ "$mode" != stop ]; then + take_over_from_leftover_daemon "$(cat "$WATCH_LOCK/pid" 2>/dev/null || true)" \ + "$(cat "$WATCH_LOCK/pid-identity" 2>/dev/null || true)" +fi + if [ "$mode" = restart ]; then stop_home_watcher || exit 1 fi @@ -696,6 +751,7 @@ owned_child_finished() { if [ "$rc" -eq 0 ]; then if wait_for_healthy_successor; then + take_over_from_leftover_daemon "$HEALTHY_PID" "$HEALTHY_IDENTITY" cycle_log_append "$rc" "$signal" unexpected-clean-exit "attached:$HEALTHY_PID" print_watch_output "$child_out" rm -f "$child_out" 2>/dev/null || true diff --git a/bin/fm-watch-checkpoint.sh b/bin/fm-watch-checkpoint.sh index 0e238d1ff9b..4b9224967b8 100755 --- a/bin/fm-watch-checkpoint.sh +++ b/bin/fm-watch-checkpoint.sh @@ -16,6 +16,12 @@ # "supervision-host:" line other than the park boundary passes through as a # wake; the boundary alone is the ordinary quiet checkpoint. On a home that # does not run the host nothing below changes. +# +# LEFTOVER AWAY DAEMON. Before the supervision host or the plain bounded +# watcher runs, an away daemon that still runs this home's watcher while +# state/.afk is absent outlived away mode and would take every wake, so the +# checkpoint stops it and this home's watcher (bin/fm-afk-daemon-lib.sh proves +# the ownership), or fails loudly. set -u SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -113,6 +119,24 @@ positive_or() { # case "$1" in ''|0*|*[!0-9]*) printf '%s\n' "$2" ;; *) printf '%s\n' "$1" ;; esac } +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-afk-daemon-lib.sh +. "$SCRIPT_DIR/fm-afk-daemon-lib.sh" +if [ ! -e "$STATE/.afk" ] \ + && DAEMON=$(fm_afk_daemon_watcher_owner "$STATE" "$SCRIPT_DIR/fm-watch.sh" "$FM_HOME"); then + if ! SURVIVORS=$(fm_afk_daemon_stop "$DAEMON"); then + echo "checkpoint: leftover away daemon pid=${SURVIVORS//$'\n'/,} did not stop, so it still owns this home's watcher" >&2 + exit 1 + fi + if ! "$SCRIPT_DIR/fm-watch-arm.sh" --stop >"$OUT" 2>"$ERR"; then + [ ! -s "$OUT" ] || cat "$OUT" + [ ! -s "$ERR" ] || cat "$ERR" >&2 + echo "checkpoint: could not stop the watcher of leftover away daemon pid=$DAEMON" >&2 + exit 1 + fi +fi + # shellcheck source=bin/fm-supervision-engine-lib.sh . "$SCRIPT_DIR/fm-supervision-engine-lib.sh" if fm_supervision_host_enabled "$CONFIG" codex; then diff --git a/docs/architecture.md b/docs/architecture.md index 4b2b6f9cbfe..9d6f18d7819 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -196,6 +196,8 @@ While the away record exists neither supervisor rechecks an item held for the ca On Pi and pi-signed the away daemon is no longer launched: the ordinary supervision session continues under the record with main parked, so the supervision branch takes every actionable wake, captain outcomes accumulate for the return brief, and main's standing authority relocates to the branch through the guarded scripts, each keeping its own gate ([`pi-supervision-branch.md`](pi-supervision-branch.md#postures)); a wake the branch cannot take and a watcher failure still reach main. On a non-Pi home that runs the [supervision host](supervision-host.md), the host runs the away session instead of the daemon. A presence-gated sub-supervisor (`bin/fm-supervise-daemon.sh`) still extends walk-away supervision on the remaining harnesses: the `/afk` skill starts it through the tracked foreground helper `bin/fm-afk-start.sh` once the record exists, after which the watcher reverts to daemon-managed one-shot mode and the daemon self-handles routine wakes in bash. +The daemon lives only as long as `state/.afk` and its singleton lock: it exits on its own, before it drains another wake, once either is gone, because a daemon left running drains and acknowledges every wake into a buffer that nothing delivers while attended. +`bin/fm-afk-daemon-lib.sh` owns finding a home's live daemons through the lock or the watcher a daemon runs, including one whose lock is gone or whose lock identity no longer matches it, for the return's shutdown and for the arm and the Codex checkpoint (`bin/fm-watch-checkpoint.sh`, also before the supervision host runs), which stop a leftover daemon that still runs the watcher while `state/.afk` is absent instead of attaching to its cycle. The watcher and daemon share `bin/fm-classify-lib.sh` for captain-relevant status verbs, declared-wait vocabulary (a `paused:` external wait and a verified `captain-held` transfer alike, through one combined predicate), and status-scan primitives. Terminal verbs remain captain-relevant, while a nonterminal progress verb cannot become terminal merely because its prose contains a legacy free-text token such as `merged`; bare legacy free-text lines remain compatible. The shared latest-event read takes the most recent line that leads with a recognized verb, a legacy token, or an unrecognized status prefix, so a bad declaration stays visible as itself while continuation prose and trailing blank lines after a multi-line record cannot hide a declared wait. diff --git a/docs/scripts.md b/docs/scripts.md index 8d656ec203e..48758094709 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -83,6 +83,7 @@ The shared no-mistakes gate lifecycle boundary is summarized in [architecture.md | `fm-secondmate-report.sh` | Optional helper that resolves the parent channel itself and appends a correlated status or document-pointer report | | `fm-extension.mjs` | Bind, inspect, verify, and strictly invoke trusted external process-event adapter packages | | `fm-extension-launch-barrier.mjs` | Publish one exact static core-owned invocation group before package code runs | +| `fm-pid-identity.mjs` | The extension host's mirror of the legacy-identity rule that `fm-pid-identity-lib.sh` owns | | `fm-extension.sh` | Expose extension binding commands through the tracked shell and remote-home command boundary | | `fm-procevent.sh` | Register, supervise, capture, classify, acknowledge, and safely retire built-in or explicitly bound process-event sources | | `fm-procevent-remote-reply.sh` | Relay the remote-secondmate status stream through non-destructive process-event deltas | @@ -97,6 +98,7 @@ The shared no-mistakes gate lifecycle boundary is summarized in [architecture.md | `fm-afk-start.sh` | Run the common sourceable away-mode daemon entry in the foreground | | `fm-afk-launch.sh` | Own away/quiet entry (same-turn record write, then read-back), exit, rollback, and any backend terminal lifecycle | | `fm-afk-return.sh` | Own deterministic return shutdown, the return brief, catch-up evidence, and the firstmate-actionable blocker gate | +| `fm-afk-daemon-lib.sh` | Find a home's live away-mode daemons, including one without its lock, through its lock or the watcher it runs | | `fm-supervisor-target-lib.sh` | Resolve the shared supervisor target and backend for the daemon and launcher | | `fm-supervise-daemon.sh` | Presence-gated away-mode sub-supervisor: self-handle routine wakes, guard injection by the detected primary harness, escalate batched digests, alert on failed delivery | | `fm-crew-state.sh` | Print one deterministic current-state line for a crew | @@ -118,6 +120,7 @@ The shared no-mistakes gate lifecycle boundary is summarized in [architecture.md | `fm-wake-grant.sh` | Serialize Pi supervision-branch wake-row claim activation, publication, release, and deactivation | | `fm-wake-lib.sh` | Shared durable wake queue, recovery generations, portable locks, and watcher identity/health helpers | | `fm-path-lib.sh` | Fork-free `dirname`/`basename` equivalents with no source-time side effects | +| `fm-pid-identity-lib.sh` | Process identities that no host time-zone change moves, and the one rule for matching a recorded identity to a live pid, with no source-time side effects | | `fm-classify-lib.sh` | Shared wake classification, durable keyed-decision folds and scans, unread status selection, home-owned status-append ranges, and bounded latest-event snapshots | | `fm-send.sh` | Steer a task via a durable inbox record plus doorbell, or send a supported key or typed harness invocation through the recorded backend | | `fm-branch-prompt.sh` | Emit the shared supervision branch's byte-stable system prompt ([pi-supervision-branch.md](pi-supervision-branch.md), [supervision-host.md](supervision-host.md)) | diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 9d9e19dadd5..07293f80b31 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -365,6 +365,7 @@ Branch acknowledgement retiring the check-row receipts of exactly its granted se An attached arm follows verified identity-matched successors and resolves the same way when that chain ends without one. It does this because it holds no handle on the watcher's stdout and cannot read the reason line itself. +The arm never attaches to a watcher whose parent is an away daemon while `state/.afk` is absent: before any health or beacon check, it stops that leftover daemon and this home's watcher, then owns a fresh cycle, or prints a typed `watcher: FAILED - ... leftover away daemon ...` line when the daemon survives ([architecture.md](architecture.md#event-driven-supervision)). ### Terminal-delivery ledger diff --git a/tests/fm-afk-launch.test.sh b/tests/fm-afk-launch.test.sh index 4cf07950437..b7891064c57 100755 --- a/tests/fm-afk-launch.test.sh +++ b/tests/fm-afk-launch.test.sh @@ -20,6 +20,8 @@ set -u ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=tests/pid-identity-helpers.sh +. "$ROOT/tests/pid-identity-helpers.sh" LAUNCH="$ROOT/bin/fm-afk-launch.sh" START="$ROOT/bin/fm-afk-start.sh" CONTRACT="$ROOT/bin/fm-afk-contract.sh" @@ -566,6 +568,93 @@ unit_stop_ordering() { rm -rf "$st" } +# Run for home from the west zone, with the ps identity path. +in_west() { # + local st=$1 + shift + TZ="$FM_TEST_TZ_WEST" FM_PROC_ROOT_OVERRIDE="$st/no-proc" FM_HOME="$st" FM_STATE_OVERRIDE="$st/state" "$@" +} + +# --------------------------------------------------------------------------- +# UNIT: a host time-zone change between recording the daemon's lock and +# reading it, the trigger of the 2026-10-01 lost worker events. An entry from +# the new zone must still see the live daemon, the turn-end ownership check +# must still credit it, and the return must still stop it rather than skip it. +# A record a build before the UTC pin wrote in local time must behave the same. +# --------------------------------------------------------------------------- +unit_time_zone_change_keeps_the_live_daemon() { + local form st lock marker daemon_pid + for form in current legacy; do + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-tz-daemon.XXXXXX") + mkdir -p "$st/state" + date '+%s' > "$st/state/.afk" + marker="$st/daemon-term" + bash -c 'trap "echo term > \"$1\"; exit 0" TERM; while :; do sleep 0.2; done' _ "$marker" & + # shellcheck disable=SC2031 # The background PID is captured immediately in this shell. + daemon_pid=$! + fm_test_wait_exec "$daemon_pid" "while :" || fail "time zone ($form): the fake daemon never started" + lock="$st/state/.supervise-daemon.lock" + mkdir -p "$lock" + printf '%s' "$daemon_pid" > "$lock/pid" + fm_test_identity_record "$daemon_pid" "$form" "$FM_TEST_TZ_EAST" "$st/no-proc" > "$lock/pid-identity" + # shellcheck disable=SC2016 # The child shell expands its own positional parameters. + if in_west "$st" bash -c '. "$1"; set +e; daemon_lock_held_by_live_daemon' _ "$START"; then + pass "time zone ($form): an entry after a zone change still sees the live daemon" + else + fail "time zone ($form): an entry after a zone change read the live daemon as dead" + fi + # shellcheck disable=SC2016 # The child shell expands its own positional parameters. + if in_west "$st" bash -c '. "$1"; fm_afk_daemon_owns_supervision "$2"' _ "$ROOT/bin/fm-wake-lib.sh" "$st/state"; then + pass "time zone ($form): the turn-end check still credits the live daemon" + else + fail "time zone ($form): the turn-end check stopped crediting the live daemon" + fi + printf 'none\t-\tnative\n' > "$st/state/.afk-daemon-terminal" + in_west "$st" "$LAUNCH" stop >/dev/null 2>&1 + # shellcheck disable=SC2031 # The background daemon writes this shared file; no shell variable is reassigned. + if [ "$(cat "$marker" 2>/dev/null || echo missing)" = term ]; then + pass "time zone ($form): the return after a zone change stopped the live daemon" + else + fail "time zone ($form): the return after a zone change skipped the live daemon" + fi + kill "$daemon_pid" 2>/dev/null || true + wait "$daemon_pid" 2>/dev/null || true + rm -rf "$st" + done +} + +# The launch lock serializes away-mode entry and return under the same identity. +unit_time_zone_change_keeps_the_launch_lock_owner() { + local form st lock owner + for form in current legacy; do + st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-tz-launch.XXXXXX") + mkdir -p "$st/state" + sleep 600 & + # shellcheck disable=SC2031 # The background PID is captured immediately in this shell. + owner=$! + fm_test_wait_exec "$owner" "sleep 600" || fail "time zone ($form): the launch lock owner never started" + lock="$st/state/.afk-launch.lock" + mkdir -p "$lock" + printf '%s' "$owner" > "$lock/pid" + printf '%s' "$(fm_test_identity_record "$owner" "$form" "$FM_TEST_TZ_EAST" "$st/no-proc")" > "$lock/pid-identity" + # shellcheck disable=SC2016 # The child shell expands its own positional parameters. + if in_west "$st" bash -c '. "$1"; set +e; fm_afk_launch_lock_owned' _ "$LAUNCH"; then + pass "time zone ($form): the launch lock keeps its live owner across a zone change" + else + fail "time zone ($form): a zone change made the launch lock's live owner read as gone" + fi + kill "$owner" 2>/dev/null || true + wait "$owner" 2>/dev/null || true + # shellcheck disable=SC2016 # The child shell expands its own positional parameters. + if in_west "$st" bash -c '. "$1"; set +e; fm_afk_launch_lock_owned' _ "$LAUNCH"; then + fail "time zone ($form): the launch lock kept a dead owner" + else + pass "time zone ($form): the launch lock releases a dead owner" + fi + rm -rf "$st" + done +} + unit_stop_rejects_reused_pid() { local st lock sleeper_pid st=$(mktemp -d "${TMPDIR:-/tmp}/fm-afk-pid-reuse.XXXXXX") @@ -1525,6 +1614,258 @@ unit_stop_confirms_daemon_exit() { rm -rf "$st" } +# A stand-in away daemon: a live process whose command line runs a script named +# fm-supervise-daemon.sh, so a lock identity that no longer matches cannot be +# mistaken for a reused pid. It exits on TERM, as the real daemon does, and +# stops itself after a bound so an escaped copy cannot outlive the suite. +make_daemon_standin() { # + mkdir -p "$1" + cat > "$1/fm-supervise-daemon.sh" <<'SH' +#!/usr/bin/env bash +trap 'exit 0' TERM +end=$(( $(date +%s) + 120 )) +while [ "$(date +%s)" -lt "$end" ]; do sleep 0.2; done +SH + chmod +x "$1/fm-supervise-daemon.sh" + printf '%s\n' "$1/fm-supervise-daemon.sh" +} + +# Start a stand-in and wait until its exec chain settles, so an identity the +# fixture records matches the one the stand-in keeps; the real daemon records +# its own identity once it runs. Sets STANDIN_PID in this shell. +STANDIN_PID= +start_daemon_standin() { #