Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
2ad05be
feat(bin): per-home remote transport lanes with cancellation, bounded…
kunchenguid Aug 27, 2026
e9e3fc3
no-mistakes(review): Protect live stages and validate send budgets early
kunchenguid Aug 27, 2026
8b839d2
no-mistakes(review): Preserve sequence lock ownership during stale re…
kunchenguid Aug 27, 2026
0896cf1
no-mistakes(review): Allocate job sequences at publication boundary
kunchenguid Aug 27, 2026
cedf157
no-mistakes(review): Bound remote keys and extend stale lock recovery
kunchenguid Aug 27, 2026
148cb77
no-mistakes(document): Document bounded remote transport behavior
kunchenguid Aug 27, 2026
31df40a
no-mistakes(lint): Suppress intentional deferred-expansion lint warning
kunchenguid Aug 28, 2026
1b23561
no-mistakes(ci): Fixed stale sequence-lock recovery by reconciling th…
kunchenguid Aug 28, 2026
5a5896b
no-mistakes(review): Use atomic sequence claims and lossless lane keys
kunchenguid Aug 28, 2026
3bddb2e
no-mistakes(review): Recover regressed sequence hints and rate-limit …
kunchenguid Aug 28, 2026
04cbd60
no-mistakes(review): Restrict worker heartbeats to serving loop
kunchenguid Aug 28, 2026
4749bed
no-mistakes(review): Verify supervisor identity before lane recovery …
kunchenguid Aug 28, 2026
caec453
no-mistakes(review): Verify tracked lane and claim owner identities
kunchenguid Aug 28, 2026
b475bdd
no-mistakes(document): Clarify remote lane and transport contracts
kunchenguid Aug 28, 2026
517381e
no-mistakes(ci): Fixed the CI time-boundary failure by pinning fm-pub…
kunchenguid Aug 28, 2026
6ff3455
no-mistakes(review): Preserve assigned lane ownership of queued jobs
kunchenguid Aug 28, 2026
8c11a30
no-mistakes(review): Reserve homes owned by foreign queued lanes
kunchenguid Aug 28, 2026
cd24070
no-mistakes(review): Preserve completed results during crash recovery
kunchenguid Aug 28, 2026
28150ae
no-mistakes(review): Harden claim cleanup, expiry, and cancellation r…
kunchenguid Aug 28, 2026
58d64ad
no-mistakes(review): Verify process groups and reap abandoned results
kunchenguid Aug 28, 2026
15c4113
no-mistakes(review): Stop leaderless groups and reap cancelled public…
kunchenguid Aug 28, 2026
b0ebf65
no-mistakes(document): Correct remote transport lifecycle documentation
kunchenguid Aug 28, 2026
9e16f42
no-mistakes(lint): Quote done state comparisons for ShellCheck
kunchenguid Aug 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bin/fm-backlog-handoff.sh
Original file line number Diff line number Diff line change
Expand Up @@ -549,7 +549,7 @@ remote_deliver_outbox() { # <secondmate-id> <outbox-path>
mv -f -- "$counter_tmp" "$counter" \
|| { rm -f -- "$snapshot" "$counter_tmp"; return 1; }
remote_rel="state/handoff/$id.outbox.md"
if ! "$SCRIPT_DIR/fm-on.sh" "$id" fm-remote-file.sh put "$remote_rel" 1048576 \
if ! "$SCRIPT_DIR/fm-on.sh" --stdin "$id" fm-remote-file.sh put "$remote_rel" 1048576 \
"$bytes" "$hash" "$generation" < "$snapshot"; then
rm -f -- "$snapshot"
echo "error: handoff transfer to $id was unavailable or completion is unknown; outbox preserved at $outbox" >&2
Expand Down
39 changes: 26 additions & 13 deletions bin/fm-on.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# Execute one tracked Firstmate command in a configured remote secondmate home.
#
# Usage:
# fm-on.sh <secondmate-id|unambiguous-ssh-alias> <fm-command> [args...]
# fm-on.sh [--stdin] <secondmate-id|unambiguous-ssh-alias> <fm-command> [args...]
#
# Routes come only from remote records in data/secondmates.md. A record names an
# SSH config alias, remote Firstmate code root, and remote FM_HOME. A host alias
Expand All @@ -11,11 +11,14 @@
# bin/fm-*.sh namespace. No per-command table exists.
#
# argv is encoded as one NUL-delimited stream and passed through the fixed
# fm-remote-entrypoint.sh. stdin remains the caller's stdin, stdout and stderr
# remain separate, and ssh's exit status is returned unchanged. OpenSSH never
# receives an auto-retry instruction here. Exit 255 therefore means unavailable
# transport or unknown remote completion and must be reconciled by the semantic
# caller, never blindly repeated by this layer.
# fm-remote-entrypoint.sh. The remote command's stdin is /dev/null by default,
# because remote staging captures stdin to EOF and an open caller stream would
# block staging indefinitely; a payload caller passes --stdin to forward its
# own stream as the job's bounded input. stdout and stderr remain separate, and
# ssh's exit status is returned unchanged. OpenSSH never receives an auto-retry
# instruction here. Exit 255 therefore means unavailable transport or unknown
# remote completion and must be reconciled by the semantic caller, never
# blindly repeated by this layer.
#
# The SSH alias keeps normal public-key and strict host-key policy in ~/.ssh.
# This command explicitly disables agent forwarding, forwarding setup, and
Expand All @@ -42,12 +45,17 @@ PROTOCOL=1
. "$SCRIPT_DIR/fm-secondmate-registry-lib.sh"

die() { printf 'error: %s\n' "$1" >&2; exit 1; }
usage() { sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }
usage() { sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }

encode_base64() {
base64 | tr -d '\n'
}

STDIN_MODE=closed
if [ "${1:-}" = --stdin ]; then
STDIN_MODE=caller
shift
fi
[ "$#" -ge 2 ] || usage
ROUTE=$1
COMMAND=$2
Expand Down Expand Up @@ -103,10 +111,15 @@ case "$ALIVE_COUNT_MAX" in ''|*[!0-9]*) die "FM_SSH_ALIVE_COUNT_MAX must be a po
[ "$ALIVE_INTERVAL" -gt 0 ] || die "FM_SSH_ALIVE_INTERVAL must be a positive integer: $ALIVE_INTERVAL"
[ "$ALIVE_COUNT_MAX" -gt 0 ] || die "FM_SSH_ALIVE_COUNT_MAX must be a positive integer: $ALIVE_COUNT_MAX"

"$SSH_BIN" \
-o ForwardAgent=no \
-o ClearAllForwardings=yes \
-o 'SendEnv=-*' \
-o "ServerAliveInterval=$ALIVE_INTERVAL" \
-o "ServerAliveCountMax=$ALIVE_COUNT_MAX" \
SSH_ARGS=(
-o ForwardAgent=no
-o ClearAllForwardings=yes
-o 'SendEnv=-*'
-o "ServerAliveInterval=$ALIVE_INTERVAL"
-o "ServerAliveCountMax=$ALIVE_COUNT_MAX"
-- "$HOST" fm-remote-entrypoint.sh "$PROTOCOL" "$ROOT_B64" "$HOME_B64" "$ARGV_B64"
)
if [ "$STDIN_MODE" = caller ]; then
exec "$SSH_BIN" "${SSH_ARGS[@]}"
fi
exec "$SSH_BIN" "${SSH_ARGS[@]}" < /dev/null
43 changes: 42 additions & 1 deletion bin/fm-remote-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,15 @@
# disconnect remains unknown completion to fm-on.sh, which preserves OpenSSH's
# exit 255 behavior. The shared library header owns job fields, bounds, PATH,
# LaunchAgent contract, and worker environment.
#
# A staged job whose caller goes away is cancelled rather than abandoned: any
# exit after staging and before the published result marks the job cancelled
# (signal traps cover a delivered HUP/TERM/PIPE/INT, and the exit trap covers a
# failed bounded wait), and while waiting this process probes its parent about
# once per second, so an ssh channel that dies without delivering any signal -
# sshd exiting and reparenting this process - also cancels the job. The worker
# then skips or stops the cancelled job instead of running it to completion for
# nobody.
set -eu

PROTOCOL=1
Expand Down Expand Up @@ -74,7 +83,36 @@ sha256_file() { # <path>
[ "$#" -eq 4 ] || die "remote entrypoint expects protocol, root, home, and argv"
[ "$1" = "$PROTOCOL" ] || die "incompatible remote protocol: local=$1 remote=$PROTOCOL"
TMP=$(mktemp -d "${TMPDIR:-/tmp}/fm-remote-entrypoint.XXXXXX") || die "cannot create protocol staging directory" 70
trap 'rm -rf -- "$TMP"' EXIT

JOB_ID=
JOB_COMPLETED=0
ACCOUNT_HOME=
ENTRYPOINT_PPID=$(ps -o ppid= -p $$ 2>/dev/null | tr -d ' ' || true)

# The recorded parent is the ssh session process; when it disappears this
# process is reparented and the caller is provably gone. An unreadable probe
# never cancels: only an observed parent change does.
# shellcheck disable=SC2329 # Invoked by fm_remote_job_wait through FM_REMOTE_JOB_DISCONNECT_PROBE.
entrypoint_caller_connected() {
local current
case "$ENTRYPOINT_PPID" in ''|*[!0-9]*) return 0 ;; esac
current=$(ps -o ppid= -p $$ 2>/dev/null | tr -d ' ' || true)
case "$current" in ''|*[!0-9]*) return 0 ;; esac
[ "$current" = "$ENTRYPOINT_PPID" ]
}

# shellcheck disable=SC2329 # Invoked through the EXIT trap below.
entrypoint_cleanup() {
rm -rf -- "$TMP"
if [ -n "$JOB_ID" ] && [ "$JOB_COMPLETED" -eq 0 ] && [ -n "$ACCOUNT_HOME" ]; then
fm_remote_job_cancel "$ACCOUNT_HOME" "$JOB_ID" 2>/dev/null || true
fi
}
trap entrypoint_cleanup EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 141' PIPE
trap 'exit 143' TERM

decode_text "remote root" "$2" "$TMP/root"
decode_text "remote home" "$3" "$TMP/home"
Expand Down Expand Up @@ -138,11 +176,14 @@ if ! fm_remote_job_ensure_worker "$ROOT" "$ACCOUNT_HOME"; then
die "${FM_REMOTE_JOB_ERROR:-remote job worker is unavailable; run fm-on.sh <route> fm-remote-doctor.sh --fix}"
fi
if ! JOB_ID=$(fm_remote_job_stage "$ACCOUNT_HOME" "$ROOT" "$HOME_PATH" "$COMMAND" "${ARGV[@]:1}"); then
JOB_ID=
die "${FM_REMOTE_JOB_ERROR:-cannot stage remote job}" 70
fi
FM_REMOTE_JOB_DISCONNECT_PROBE=entrypoint_caller_connected
if ! fm_remote_job_wait "$ACCOUNT_HOME" "$JOB_ID"; then
die "${FM_REMOTE_JOB_ERROR:-remote job did not complete}" 70
fi
JOB_COMPLETED=1
cat "$FM_REMOTE_JOB_STDOUT"
cat "$FM_REMOTE_JOB_STDERR" >&2
RESULT=$FM_REMOTE_JOB_EXIT
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-remote-home-seed.sh
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ if [ "$PREFLIGHT_RC" -ne 0 ]; then
fi

set +e
PROVISION_OUT=$("$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-home-provision.sh < "$TMP/manifest" 2>&1)
PROVISION_OUT=$("$SCRIPT_DIR/fm-on.sh" --stdin "$ID" fm-remote-home-provision.sh < "$TMP/manifest" 2>&1)
PROVISION_RC=$?
set -e
if [ "$PROVISION_RC" -ne 0 ]; then
Expand Down
2 changes: 1 addition & 1 deletion bin/fm-remote-inherit-push.sh
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ while IFS= read -r rel; do
[ -f "$snapshot" ] && [ ! -L "$snapshot" ] || die "inherited source snapshot is unsafe: $source"
bytes=$(LC_ALL=C wc -c < "$snapshot" | tr -d ' ')
hash=$(sha256_file "$snapshot") || die "cannot hash inherited source: $source"
"$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-inherit.sh put "$rel" "$bytes" "$hash" "$GENERATION" < "$snapshot"
"$SCRIPT_DIR/fm-on.sh" --stdin "$ID" fm-remote-inherit.sh put "$rel" "$bytes" "$hash" "$GENERATION" < "$snapshot"
else
# This loop's heredoc is its control stream, not remote command input.
"$SCRIPT_DIR/fm-on.sh" "$ID" fm-remote-inherit.sh absent "$rel" 0 "$EMPTY_HASH" "$GENERATION" < /dev/null
Expand Down
Loading
Loading