diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9fff23c2b4..39d32d0859a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,6 +45,7 @@ jobs: run: | set -eu bash tests/fm-worker-isolation.test.sh + bash tests/fm-inactive-outcome.test.sh bash tests/fm-watch-session.test.sh bash tests/fm-slot-occupant-proof.test.sh FM_TEARDOWN_TEST_FOCUS=s1 bash tests/fm-teardown.test.sh diff --git a/AGENTS.md b/AGENTS.md index e9cc9c72ed4..5c8273ff6df 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -81,11 +81,13 @@ data/ personal fleet records; LOCAL, gitignored as a whole /brief.md per-task crewmate brief, or per-secondmate charter brief when kind=secondmate /report.md scout task deliverable, written by the crewmate; survives teardown projects/ cloned repos; gitignored; READ-ONLY for you -state/ volatile runtime signals; gitignored +state/ gitignored operational state: volatile runtime signals plus durable local inactive-outcome receipts .status appended by crewmates: ": " wake-event lines, not current-state truth .turn-ended touched by turn-end hooks - .meta written by fm-spawn: window=, worktree=, project=, harness=, kind=, mode=, yolo=; kind=secondmate also records home= and projects= (fm-pr-check appends pr= and verified pr_head= when available) + .meta written by fm-spawn: window=, worktree=, project=, harness=, kind=, mode=, yolo=, and spawn_incarnation=; no-mistakes ship tasks also carry the run-binding handoff/state and, after binding, run_id=; kind=secondmate also records home= and projects= (fm-pr-check appends pr= and verified pr_head= when available) .check.sh optional slow poll you write per task (e.g. merged-PR check) + terminal-outcomes/ fingerprinted local inactive terminal-outcome receipts; the locked wake drain is the only path that presents and acknowledges them + .inactive-outcome-* bounded scan, claim, cursor, and retry state; never touch x-watch.check.sh generated X-mode relay poll shim; present only when opted in (section 14) x-inbox/ generated X-mode pending mention payloads; fmx-respond drains it (section 14) x-outbox/ generated X-mode dry-run reply previews; inspect it when FMX_DRY_RUN is set (section 14) @@ -393,7 +395,7 @@ If one pair fails, the rest still run and the batch exits non-zero. The script resolves the harness (`fm-harness.sh crew`), owns the verified launch templates, resolves the project's delivery mode (`fm-project-mode.sh`) for ship/scout tasks, and records `harness=`, `kind=`, `mode=`, and `yolo=` in the task's meta; a non-flag third argument containing whitespace is treated as a raw launch command (only for verifying new adapters). For `kind=secondmate`, the same script launches in the registered or explicit firstmate home instead of running `treehouse get` for a project, records `home=` and `projects=`, and uses the charter brief as the launch prompt. -For ship and scout tasks, the script creates the window (in your current tmux session, or a dedicated `firstmate` session when you are outside tmux), runs `treehouse get`, waits for the worktree subshell, asserts the resolved worktree is a genuine isolated worktree distinct from the primary checkout (aborting the spawn otherwise, to prevent the worktree tangle of section 8), installs the turn-end hook, records `state/.meta`, and launches the agent with the brief. +For ship and scout tasks, the script creates the window (in your current tmux session, or a dedicated `firstmate` session when you are outside tmux), runs `treehouse get`, waits for the worktree subshell, asserts the resolved worktree is a genuine isolated worktree distinct from the primary checkout (aborting the spawn otherwise, to prevent the worktree tangle of section 8), installs the turn-end hook, records `state/.meta`, and launches the agent with the brief. No-mistakes ship spawns install a task-local command bridge in the child PATH; when the child invokes `no-mistakes axi run`, the bridge captures the actual returned run ID and publishes its binding under the existing task lock before the command can continue. Ship spawns never fabricate or infer a run ID, and refuse to launch if this handoff cannot be installed. For `kind=secondmate`, the script creates the same kind of window but starts directly in the persistent home. Before launching a secondmate, the script fast-forwards its home worktree to firstmate's own current default-branch commit, so a freshly spawned or recovery-respawned secondmate always starts on firstmate's current version. This is a purely local fast-forward of tracked files - never a fetch from origin, and never touching the gitignored operational dirs - so the secondmate's backlog, projects, and any prior in-flight work are untouched; a dirty, diverged, or in-flight home is left as-is and launches unchanged. diff --git a/bin/backends/herdr.sh b/bin/backends/herdr.sh index bb5b85cd1c7..5b30ac85c7f 100755 --- a/bin/backends/herdr.sh +++ b/bin/backends/herdr.sh @@ -358,6 +358,7 @@ fm_backend_herdr_server_ensure() { # local session=$1 running out i running=$(fm_backend_herdr_cli "$session" status --json 2>/dev/null | jq -r '.server.running // false' 2>/dev/null) [ "$running" = "true" ] && return 0 + [ "${FM_BACKEND_HERDR_NO_SERVER_START:-0}" = 1 ] && return 1 ( fm_backend_herdr_cli "$session" server >/dev/null 2>&1 & ) || return 1 for i in $(seq 1 20); do running=$(fm_backend_herdr_cli "$session" status --json 2>/dev/null | jq -r '.server.running // false' 2>/dev/null) diff --git a/bin/fm-crew-state.sh b/bin/fm-crew-state.sh index a46a16cd02d..e8d535c47d4 100755 --- a/bin/fm-crew-state.sh +++ b/bin/fm-crew-state.sh @@ -59,6 +59,14 @@ STATE="${FM_STATE_OVERRIDE:-$FM_HOME/state}" . "$SCRIPT_DIR/fm-backend.sh" # shellcheck source=bin/fm-numeric-lib.sh . "$SCRIPT_DIR/fm-numeric-lib.sh" +# shellcheck source=bin/fm-wake-lib.sh +FM_SESSION_LOCK_BOOTSTRAP=1 +FM_WAKE_LIB_NO_INIT=1 +. "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-run-step-lib.sh +FM_RUN_STEP_READ_ONLY=1 +. "$SCRIPT_DIR/fm-run-step-lib.sh" +trap 'fm_run_step_binding_release_held >/dev/null 2>&1 || true' EXIT ID=${1:-} [ -n "$ID" ] || { echo "usage: fm-crew-state.sh " >&2; exit 2; } @@ -368,6 +376,62 @@ nm_run_ids_for_branch() { # # CREW_BRANCH is empty at detached HEAD (a just-spawned crew, or a scout's # scratch worktree); with no branch there is no run to attribute to this crew. CREW_BRANCH=$(git -C "$WT" symbolic-ref --quiet --short HEAD 2>/dev/null || true) +META_INCARNATION=$(awk -F= '$1 == "spawn_incarnation" { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }' "$META" 2>/dev/null || true) +META_SPAWN_STATE=absent +META_SPAWN_STATE_COUNT=0 +META_BINDING_STATE=absent +META_BINDING_STATE_COUNT=0 +while IFS= read -r META_BINDING_LINE; do + case "$META_BINDING_LINE" in + spawn_state=*) + META_SPAWN_STATE_COUNT=$((META_SPAWN_STATE_COUNT + 1)) + META_SPAWN_STATE=${META_BINDING_LINE#*=} + ;; + run_binding_state=*) + META_BINDING_STATE_COUNT=$((META_BINDING_STATE_COUNT + 1)) + META_BINDING_STATE=${META_BINDING_LINE#*=} + ;; + esac +done < "$META" +if [ "$META_SPAWN_STATE_COUNT" -ne 0 ]; then + [ "$META_SPAWN_STATE_COUNT" = 1 ] || META_SPAWN_STATE=invalid + case "$META_SPAWN_STATE" in + starting|aborted) ;; + *) META_SPAWN_STATE=invalid ;; + esac +fi +if [ "$META_BINDING_STATE_COUNT" -ne 0 ]; then + [ "$META_BINDING_STATE_COUNT" = 1 ] || META_BINDING_STATE=invalid + case "$META_BINDING_STATE" in + pending|staged|bound) ;; + *) META_BINDING_STATE=invalid ;; + esac +fi +BOUND_RUN_ID= +BOUND_RUN_STATUS=75 +BOUND_RUN_REQUIRED=0 +BOUND_RUN_EVIDENCE=$(fm_run_step_binding_path "$ID") || exit 1 +if [ -e "$BOUND_RUN_EVIDENCE" ] || [ -L "$BOUND_RUN_EVIDENCE" ]; then + BOUND_RUN_REQUIRED=1 +fi +case "$META_BINDING_STATE" in + pending|bound|staged|invalid) BOUND_RUN_REQUIRED=1 ;; +esac +case "$META_SPAWN_STATE" in + starting|aborted|invalid) BOUND_RUN_REQUIRED=1 ;; +esac +if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && [ -n "$META_INCARNATION" ]; then + if fm_run_step_binding_read_held "$ID" "$META_INCARNATION"; then + BOUND_RUN_ID=$FM_RUN_STEP_HELD_VALUE + BOUND_RUN_STATUS=0 + else + BOUND_RUN_STATUS=$? + fi + [ "$BOUND_RUN_STATUS" = 1 ] && { + printf '%s\n' 'state: unknown · source: run-step · incarnation binding could not be read' >&2 + exit 1 + } +fi # A branch match is not enough when branch names are reused. Accept a run only # when its recorded head is the local worktree HEAD, or a descendant of that @@ -387,30 +451,49 @@ HAVE_RUN=0 # Scouts and secondmates never drive a no-mistakes validation of their own # worktree, so skip the lookup for them and read state from pane/log directly. if [ "$KIND" = ship ] && [ -n "$CREW_BRANCH" ] && command -v no-mistakes >/dev/null 2>&1; then - RUN_OUT=$(nm_run axi status) - if [ -n "$RUN_OUT" ]; then - run_branch=$(strip_quotes "$(nm_field branch)") - if [ -n "$run_branch" ] && [ "$run_branch" = "$CREW_BRANCH" ] && nm_run_head_matches_worktree; then - HAVE_RUN=1 + if [ "$BOUND_RUN_STATUS" = 0 ] || [ "$BOUND_RUN_REQUIRED" = 1 ]; then + BOUND_RUN_REQUIRED=1 + if [ "$BOUND_RUN_STATUS" = 0 ]; then + RUN_OUT=$(nm_run axi status --run "$BOUND_RUN_ID") + run_id=$(strip_quotes "$(nm_field id)") + run_branch=$(strip_quotes "$(nm_field branch)") + if [ "$run_id" = "$BOUND_RUN_ID" ] && [ "$run_branch" = "$CREW_BRANCH" ] \ + && nm_run_head_matches_worktree; then + HAVE_RUN=1 + else + RUN_OUT= + fi else - # The active-or-most-recent run is for another branch, or its branch name - # matches but its code identity does not. Inspect bounded recent runs for - # this branch until one is compatible with the worktree. - list_out=$(nm_run axi) - candidate_ids=$(nm_run_ids_for_branch "$CREW_BRANCH" "$list_out") - while IFS= read -r rid; do - [ -n "$rid" ] || continue - RUN_OUT=$(nm_run axi status --run "$rid") - run_branch=$(strip_quotes "$(nm_field branch)") - if [ "$run_branch" = "$CREW_BRANCH" ] && nm_run_head_matches_worktree; then - HAVE_RUN=1 - break - fi - done <<< "$candidate_ids" + RUN_OUT= + fi + else + RUN_OUT=$(nm_run axi status) + if [ -n "$RUN_OUT" ]; then + run_branch=$(strip_quotes "$(nm_field branch)") + if [ -n "$run_branch" ] && [ "$run_branch" = "$CREW_BRANCH" ] && nm_run_head_matches_worktree; then + HAVE_RUN=1 + else + # The active-or-most-recent run is for another branch, or its branch name + # matches but its code identity does not. Inspect bounded recent runs for + # this branch until one is compatible with the worktree. + list_out=$(nm_run axi) + candidate_ids=$(nm_run_ids_for_branch "$CREW_BRANCH" "$list_out") + while IFS= read -r rid; do + [ -n "$rid" ] || continue + RUN_OUT=$(nm_run axi status --run "$rid") + run_branch=$(strip_quotes "$(nm_field branch)") + if [ "$run_branch" = "$CREW_BRANCH" ] && nm_run_head_matches_worktree; then + HAVE_RUN=1 + break + fi + done <<< "$candidate_ids" + fi fi fi fi +[ "$HAVE_RUN" = 1 ] || [ "$BOUND_RUN_REQUIRED" = 0 ] || emit unknown run-step "bound run unavailable" + # --- run-step authoritative path ------------------------------------------- if [ "$HAVE_RUN" = 1 ]; then @@ -530,6 +613,38 @@ if [ "$HAVE_RUN" = 1 ]; then RUN_DETAIL="$RUN_DETAIL${SEP}convergence-round=$convergence_round${SEP}convergence-fingerprint=unavailable" fi + run_id=$(strip_quotes "$(nm_field id)") + run_step_id_count=$(grep -c '^run_step_id=' "$META" 2>/dev/null || true) + run_step_id= + if [ "$run_step_id_count" = 1 ]; then + run_step_id=$(grep '^run_step_id=' "$META" 2>/dev/null | cut -d= -f2- || true) + fi + incarnation=$(awk -F= '$1 == "spawn_incarnation" { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }' "$META" 2>/dev/null || true) + case "$RUN_STATE" in + done|failed) + case "$run_step_id_count" in + 0) run_step_id_matches_meta=1 ;; + 1) [ "$run_step_id" = "$run_id" ] && run_step_id_matches_meta=1 || run_step_id_matches_meta=0 ;; + *) run_step_id_matches_meta=0 ;; + esac + if [ "$run_step_id_matches_meta" != 1 ] || [ -z "$run_id" ] || [ -z "$incarnation" ]; then + printf '%s\n' 'state: unknown · source: run-step · incarnation binding unavailable' >&2 + exit 75 + fi + fm_run_step_binding_validate "$ID" "$run_id" "$incarnation" || { + status=$? + printf '%s\n' 'state: unknown · source: run-step · incarnation binding unavailable' >&2 + [ "$status" = 75 ] && exit 75 + exit 1 + } + ;; + esac + case "$RUN_STATE" in + working|parked|paused|done|failed) + [ -n "$run_id" ] && RUN_DETAIL="$RUN_DETAIL${SEP}run-id=$run_id" + ;; + esac + emit "$RUN_STATE" run-step "$RUN_DETAIL" fi diff --git a/bin/fm-inactive-reconcile.sh b/bin/fm-inactive-reconcile.sh new file mode 100755 index 00000000000..5fff9ce5259 --- /dev/null +++ b/bin/fm-inactive-reconcile.sh @@ -0,0 +1,4480 @@ +#!/usr/bin/env bash +# Replay an authoritative terminal crew state that stayed quiet long enough to +# be missed by the normal watcher. This is a reporting layer only: it never +# closes an endpoint, returns a slot, removes a worktree, or changes a PR. +set -u +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-worker-isolation-lib.sh +. "$SCRIPT_DIR/fm-worker-isolation-lib.sh" +fm_worker_refuse_primary_operation "inactive outcome reconciliation" || exit 1 +# shellcheck source=bin/fm-pane-idle-lib.sh +. "$SCRIPT_DIR/fm-pane-idle-lib.sh" + +FM_ROOT="${FM_ROOT_OVERRIDE:-${FM_ROOT:-$(cd "$SCRIPT_DIR/.." && pwd)}}" +FM_HOME="${FM_HOME:-${FM_ROOT_OVERRIDE:-$FM_ROOT}}" +STATE="${FM_STATE_OVERRIDE:-${STATE:-$FM_HOME/state}}" +OUTCOME_DIR="$STATE/terminal-outcomes" +SCAN_MARKER="$STATE/.inactive-outcome-reconcile" +SCAN_CURSOR="$STATE/.inactive-outcome-reconcile.cursor" +SCAN_CURSOR_IDENTITY="$STATE/.inactive-outcome-reconcile.cursor.identity" +REPORTED_ROUTE_CURSOR="$STATE/.reported-secondmate-route-repair.cursor" +PENDING_RECEIPT_CURSOR="$STATE/.pending-receipt-republish.cursor" +MAINTENANCE_PHASE_CURSOR="$STATE/.inactive-outcome-maintenance.cursor" +MAINTENANCE_ORDER_CURSOR="$STATE/.inactive-outcome-maintenance-order.cursor" +DIRECT_FIND_PENDING="$STATE/.inactive-outcome-find.pending" +DIRECT_FIND_OFFSET="$STATE/.inactive-outcome-find.pending.offset" +DIRECT_FIND_RETRY="$STATE/.inactive-outcome-find.pending.retry" +DIRECT_FIND_INCOMPLETE="$STATE/.inactive-outcome-find.incomplete" +DIRECT_FIND_ENUM_CURSOR="$STATE/.inactive-outcome-find.enum.cursor" +ENUM_PROGRESS_DIR="$STATE/.inactive-outcome-enum" +REPORTED_ROUTE_PENDING="$STATE/.reported-secondmate-route-repair.pending" +REPORTED_ROUTE_OFFSET="$STATE/.reported-secondmate-route-repair.pending.offset" +REPORTED_ROUTE_RETRY="$STATE/.reported-secondmate-route-repair.pending.retry" +REPORTED_ROUTE_ENUM_CURSOR="$STATE/.reported-secondmate-route-repair.enum.cursor" +PENDING_RECEIPT_PENDING="$STATE/.pending-receipt-republish.pending" +PENDING_RECEIPT_OFFSET="$STATE/.pending-receipt-republish.pending.offset" +PENDING_RECEIPT_RETRY="$STATE/.pending-receipt-republish.pending.retry" +PENDING_RECEIPT_ENUM_CURSOR="$STATE/.pending-receipt-republish.enum.cursor" +MERGE_TXN="$STATE/.inactive-outcome-merge.txn" +FM_WAKE_QUEUE="${FM_WAKE_QUEUE:-$STATE/.wake-queue}" +PANE_IDLE_DIR="$STATE/.pane-idle" +PANE_IDLE_INDEX_DIR="$STATE/.inactive-outcome-pane-idle-index" + +inactive_state_path_is_safe() { + local path=$1 kind=$2 + [ ! -L "$path" ] || return 1 + if [ -e "$path" ]; then + case "$kind" in + dir) [ -d "$path" ] || return 1 ;; + file) [ -f "$path" ] || return 1 ;; + *) return 1 ;; + esac + fi +} + +inactive_state_preflight() { + if [ -L "$STATE" ] || { [ -e "$STATE" ] && [ ! -d "$STATE" ]; }; then + return 1 + fi + if [ ! -e "$STATE" ] && ! mkdir -p "$STATE"; then + return 1 + fi + inactive_state_path_is_safe "$STATE" dir || return 1 + inactive_state_path_is_safe "$OUTCOME_DIR" dir || return 1 + inactive_state_path_is_safe "$SCAN_MARKER" file || return 1 + inactive_state_path_is_safe "$SCAN_CURSOR" file || return 1 + inactive_state_path_is_safe "$SCAN_CURSOR_IDENTITY" file || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_CURSOR" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_CURSOR" file || return 1 + inactive_state_path_is_safe "$MAINTENANCE_PHASE_CURSOR" file || return 1 + inactive_state_path_is_safe "$MAINTENANCE_ORDER_CURSOR" file || return 1 + inactive_state_path_is_safe "$DIRECT_FIND_PENDING" file || return 1 + inactive_state_path_is_safe "$DIRECT_FIND_OFFSET" file || return 1 + inactive_state_path_is_safe "$DIRECT_FIND_RETRY" file || return 1 + inactive_state_path_is_safe "$DIRECT_FIND_INCOMPLETE" file || return 1 + inactive_state_path_is_safe "$DIRECT_FIND_ENUM_CURSOR" file || return 1 + inactive_state_path_is_safe "$ENUM_PROGRESS_DIR" dir || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_PENDING" file || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_OFFSET" file || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_RETRY" file || return 1 + inactive_state_path_is_safe "$REPORTED_ROUTE_ENUM_CURSOR" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_PENDING" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_OFFSET" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_RETRY" file || return 1 + inactive_state_path_is_safe "$PENDING_RECEIPT_ENUM_CURSOR" file || return 1 + inactive_state_path_is_safe "$MERGE_TXN" file || return 1 + inactive_state_path_is_safe "$PANE_IDLE_DIR" dir || return 1 + inactive_state_path_is_safe "$PANE_IDLE_INDEX_DIR" dir || return 1 + inactive_state_path_is_safe "$FM_WAKE_QUEUE" file || return 1 +} + +inactive_state_preflight || { + echo "error: inactive reconciliation state must be local regular state under $FM_HOME" >&2 + exit 1 +} + +# shellcheck source=bin/fm-wake-lib.sh +. "$SCRIPT_DIR/fm-wake-lib.sh" +# shellcheck source=bin/fm-backend.sh +. "$SCRIPT_DIR/fm-backend.sh" +# shellcheck source=bin/fm-pending-reply-lib.sh +. "$SCRIPT_DIR/fm-pending-reply-lib.sh" + +FM_CREW_STATE_BIN="${FM_CREW_STATE_BIN:-$SCRIPT_DIR/fm-crew-state.sh}" +SCAN_LOCK="$STATE/.inactive-outcome-reconcile.lock" +ROUTE_MARKER="$FM_HOME/.fm-secondmate-home" +CHILD_LOCK_HELD=0 +CHILD_LOCK= +CHILD_LOCK_OWNER= +WAKE_QUEUE_LOCK_HELD=0 +MAINTENANCE_ITEMS_PROCESSED=0 + +wake_queue_lock_acquire() { + [ "$WAKE_QUEUE_LOCK_HELD" = 1 ] && return 0 + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 1 + WAKE_QUEUE_LOCK_HELD=1 +} + +wake_queue_lock_release() { + [ "$WAKE_QUEUE_LOCK_HELD" = 1 ] || return 0 + WAKE_QUEUE_LOCK_HELD=0 + fm_lock_release "$FM_WAKE_QUEUE_LOCK" +} + +bounded_secs() { + local value=$1 fallback=$2 minimum=$3 maximum=$4 value_len minimum_len maximum_len LC_ALL=C + case "$value" in ''|*[!0-9]*) value=$fallback ;; esac + while [ "${value#0}" != "$value" ]; do value=${value#0}; done + [ -n "$value" ] || value=0 + value_len=${#value} + minimum_len=${#minimum} + maximum_len=${#maximum} + if [ "$value_len" -lt "$minimum_len" ] \ + || { [ "$value_len" -eq "$minimum_len" ] && [ "$value" -lt "$minimum" ]; }; then + value=$minimum + elif [ "$value_len" -gt "$maximum_len" ] \ + || { [ "$value_len" -eq "$maximum_len" ] && [ "$value" -gt "$maximum" ]; }; then + value=$maximum + fi + printf '%s' "$value" +} + +clock_millis() { + local stamp + stamp=$(date +%s%N 2>/dev/null || true) + case "$stamp" in + ''|*[!0-9]*) printf '%s000' "$(date +%s)" ;; + *) + if [ "${#stamp}" -gt 10 ]; then + printf '%s' "${stamp:0:${#stamp}-6}" + else + printf '%s000' "$stamp" + fi + ;; + esac +} + +budget_remaining_secs() { + local deadline_ms=$1 now_ms remaining_ms + now_ms=$(clock_millis) + remaining_ms=$((deadline_ms - now_ms)) + if [ "$remaining_ms" -le 0 ]; then + printf '0' + else + printf '%s' "$(((remaining_ms + 999) / 1000))" + fi +} + +inactive_copy_nul_prefix() { + local source=$1 target=$2 deadline_ms=${3:-} + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + [ ! -L "$target" ] || return 1 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + command -v perl >/dev/null 2>&1 || return 125 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($source, $target, $deadline) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + my $in; + sysopen($in, $source, O_RDONLY | $nofollow) or exit 1; + my $out; + sysopen($out, $target, O_WRONLY | O_CREAT | O_TRUNC | $nofollow, 0600) or exit 1; + binmode($in); + binmode($out); + local $/ = "\0"; + while (defined(my $value = <$in>)) { + if (defined($deadline) && $deadline ne q{} && int(time() * 1000) >= $deadline) { + exit 124; + } + print($out $value) or exit 1; + } + close($in) or exit 1; + close($out) or exit 1; + ' "$source" "$target" "$deadline_ms" +} + +inactive_append_nul_records() { + local source=$1 target=$2 deadline_ms=${3:-} + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + [ -f "$target" ] && [ ! -L "$target" ] || return 1 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + command -v perl >/dev/null 2>&1 || return 125 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($source, $target, $deadline) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + my $in; + sysopen($in, $source, O_RDONLY | $nofollow) or exit 1; + my $out; + sysopen($out, $target, O_WRONLY | O_APPEND | $nofollow) or exit 1; + binmode($in); + binmode($out); + local $/ = "\0"; + while (defined(my $value = <$in>)) { + if (defined($deadline) && $deadline ne q{} && int(time() * 1000) >= $deadline) { + exit 124; + } + print($out $value) or exit 1; + } + close($in) or exit 1; + close($out) or exit 1; + ' "$source" "$target" "$deadline_ms" +} + +inactive_merge_path_is_safe() { + local path=$1 + [ -n "$path" ] || return 1 + [ "$path" != "$STATE" ] || return 1 + case "$path" in + "$STATE"/*) ;; + *) return 1 ;; + esac + case "$path" in + *$'\n'*|*$'\r'*|*$'\t'*) return 1 ;; + esac +} + +inactive_merge_txn_write() { + local source=$1 target=$2 stage=$3 tmp + inactive_merge_path_is_safe "$source" || return 1 + inactive_merge_path_is_safe "$target" || return 1 + inactive_merge_path_is_safe "$stage" || return 1 + [ "$source" != "$target" ] || return 1 + [ "$source" != "$stage" ] || return 1 + [ "$target" != "$stage" ] || return 1 + [ "$source" != "$MERGE_TXN" ] || return 1 + [ "$target" != "$MERGE_TXN" ] || return 1 + [ "$stage" != "$MERGE_TXN" ] || return 1 + [ ! -e "$MERGE_TXN" ] && [ ! -L "$MERGE_TXN" ] || return 1 + tmp=$(mktemp "$MERGE_TXN.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + if ! printf 'schema=fm-jt-inactive-merge.v1\nsource=%s\ntarget=%s\nstage=%s\n' \ + "$source" "$target" "$stage" | fm_nofollow_write "$tmp"; then + rm -f "$tmp" + return 1 + fi + [ ! -L "$MERGE_TXN" ] || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$MERGE_TXN" || { rm -f "$tmp"; return 1; } +} + +inactive_merge_txn_recover() { + local schema source target stage + if [ ! -e "$MERGE_TXN" ] && [ ! -L "$MERGE_TXN" ]; then + return 0 + fi + [ -f "$MERGE_TXN" ] && [ ! -L "$MERGE_TXN" ] || return 1 + schema=$(meta_value_unique "$MERGE_TXN" schema) || return 1 + source=$(meta_value_unique "$MERGE_TXN" source) || return 1 + target=$(meta_value_unique "$MERGE_TXN" target) || return 1 + stage=$(meta_value_unique "$MERGE_TXN" stage) || return 1 + [ "$schema" = fm-jt-inactive-merge.v1 ] || return 1 + inactive_merge_path_is_safe "$source" || return 1 + inactive_merge_path_is_safe "$target" || return 1 + inactive_merge_path_is_safe "$stage" || return 1 + [ "$source" != "$target" ] || return 1 + [ "$source" != "$stage" ] || return 1 + [ "$target" != "$stage" ] || return 1 + [ "$source" != "$MERGE_TXN" ] || return 1 + [ "$target" != "$MERGE_TXN" ] || return 1 + [ "$stage" != "$MERGE_TXN" ] || return 1 + if [ -e "$stage" ] || [ -L "$stage" ]; then + [ -f "$stage" ] && [ ! -L "$stage" ] || return 1 + [ ! -L "$target" ] || return 1 + if [ -e "$target" ]; then + [ -f "$target" ] || return 1 + fi + mv -f "$stage" "$target" || return 1 + else + [ -f "$target" ] && [ ! -L "$target" ] || return 1 + fi + if [ -e "$source" ] || [ -L "$source" ]; then + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + rm -f "$source" || return 1 + fi + rm -f "$MERGE_TXN" || return 1 +} + +inactive_append_nul_records_atomic() { + local source=$1 target=$2 deadline_ms=${3:-} tmp rc + inactive_merge_txn_recover || return 1 + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + [ -f "$target" ] && [ ! -L "$target" ] || return 1 + [ "$source" != "$target" ] || return 1 + [ ! -L "$target" ] || return 1 + tmp=$(mktemp "$target.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + inactive_append_nul_records "$target" "$tmp" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "$tmp" + return "$rc" + fi + inactive_append_nul_records "$source" "$tmp" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "$tmp" + return "$rc" + fi + inactive_merge_txn_write "$source" "$target" "$tmp" || { + rm -f "$tmp" + return 1 + } + [ ! -L "$target" ] || return 1 + mv -f "$tmp" "$target" || return 1 + rm -f "$source" || return 1 + rm -f "$MERGE_TXN" || return 1 +} + +inactive_retry_file_prepare() { + local path=$1 + if [ -e "$path" ] || [ -L "$path" ]; then + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + return 0 + fi + [ ! -L "$path" ] || return 1 + command -v perl >/dev/null 2>&1 || return 125 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + my $fh; + sysopen($fh, $path, O_WRONLY | O_CREAT | O_EXCL | $nofollow, 0600) or exit 1; + close($fh) or exit 1; + ' "$path" +} + +inactive_append_nul_value() { + local target=$1 value=$2 deadline_ms=${3-} now + [ -f "$target" ] && [ ! -L "$target" ] || return 1 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + if [ -n "$deadline_ms" ]; then + now=$(clock_millis) + [ "$now" -lt "$deadline_ms" ] || return 124 + fi + [ ! -L "$target" ] || return 1 + command -v perl >/dev/null 2>&1 || return 125 + printf '%s\0' "$value" | perl -e ' + use Fcntl qw(:DEFAULT); + my ($target) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + my $fh; + sysopen($fh, $target, O_WRONLY | O_APPEND | $nofollow) or exit 1; + binmode($fh); + local $/; + my $value = // ""; + print($fh $value) or exit 1; + close($fh) or exit 1; + ' "$target" + if [ -n "$deadline_ms" ]; then + now=$(clock_millis) + [ "$now" -lt "$deadline_ms" ] || return 124 + fi +} + +inactive_nul_last_value() { + local source=$1 deadline_ms=${2:-} value last='' + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + while IFS= read -r -d '' value; do + if [ -n "$deadline_ms" ] && [ "$(clock_millis)" -ge "$deadline_ms" ]; then + return 124 + fi + last=$value + done < "$source" + printf '%s' "$last" +} + +inactive_pending_offset() { + local path=$1 value + [ -e "$path" ] || { printf '0'; return 0; } + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + value=$(cat "$path" 2>/dev/null || true) + case "$value" in ''|*[!0-9]*) return 1 ;; esac + printf '%s' "$value" +} + +inactive_pending_offset_write() { + local path=$1 value=$2 tmp + case "$value" in ''|*[!0-9]*) return 1 ;; esac + [ ! -L "$path" ] || return 1 + tmp=$(mktemp "$path.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + if ! printf '%s\n' "$value" | fm_nofollow_write "$tmp" || [ -L "$path" ] || ! mv -f "$tmp" "$path"; then + rm -f "$tmp" + return 1 + fi +} + +inactive_text_cursor_read() { + local path=$1 value + [ -e "$path" ] || { printf ''; return 0; } + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + value=$(cat "$path" 2>/dev/null) || return 1 + case "$value" in *$'\r'*|*$'\n'*|*$'\t'*) return 1 ;; esac + printf '%s' "$value" +} + +inactive_text_cursor_write() { + local path=$1 value=$2 tmp + case "$value" in *$'\r'*|*$'\n'*|*$'\t'*) return 1 ;; esac + [ ! -L "$path" ] || return 1 + tmp=$(mktemp "$path.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + if ! printf '%s\n' "$value" | fm_nofollow_write "$tmp" || [ -L "$path" ] || ! mv -f "$tmp" "$path"; then + rm -f "$tmp" + return 1 + fi +} + +inactive_pending_clear() { + local pending=$1 offset=$2 + rm -f "$pending" "$offset" +} + +inactive_pending_retry_defer() { + local retry_source=$1 retry_target=$2 deadline_ms=${3:-} force=${4:-} incoming + [ -f "$retry_source" ] && [ ! -L "$retry_source" ] || return 1 + if [ -e "$retry_target" ] || [ -L "$retry_target" ]; then + [ -f "$retry_target" ] && [ ! -L "$retry_target" ] || return 1 + incoming="${retry_target}.incoming.$$.$RANDOM" + [ ! -e "$incoming" ] && [ ! -L "$incoming" ] || return 1 + mv -f "$retry_source" "$incoming" || return 1 + return 124 + fi + if [ "$force" = 1 ] || { [ -n "$deadline_ms" ] && [ "$(clock_millis)" -ge "$deadline_ms" ]; }; then + mv -f "$retry_source" "$retry_target" || return 1 + return 124 + fi + return 1 +} + +inactive_pending_retry_exists() { + local retry=$1 path + if [ -e "$retry" ] || [ -L "$retry" ]; then + return 0 + fi + for path in "${retry}.incoming."*; do + [ -e "$path" ] || [ -L "$path" ] || continue + return 0 + done + return 1 +} + +inactive_pending_retry_restore() { + local retry=$1 pending=$2 path source= + [ ! -e "$pending" ] && [ ! -L "$pending" ] || return 1 + if [ -e "$retry" ] || [ -L "$retry" ]; then + source=$retry + else + for path in "${retry}.incoming."*; do + [ -e "$path" ] || [ -L "$path" ] || continue + source=$path + break + done + fi + [ -n "$source" ] || return 1 + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + mv -f "$source" "$pending" +} + +inactive_nul_contains_from_offset() { + local pending=$1 offset=$2 needle=$3 deadline_ms=${4:-} value + [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 + case "$offset" in ''|*[!0-9]*) return 1 ;; esac + while IFS= read -r -d '' value; do + if [ -n "$deadline_ms" ] && [ "$(clock_millis)" -ge "$deadline_ms" ]; then + return 124 + fi + if [ "$offset" -gt 0 ]; then + offset=$((offset - 1)) + continue + fi + [ "$value" = "$needle" ] && return 0 + done < "$pending" + return 1 +} + +inactive_pending_retry_filter_duplicates() { + local pending=$1 retry=$2 deadline_ms='' offset tags sorted tmp rc remaining tab + [ "$#" -ge 3 ] && deadline_ms=$3 + [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 + [ -f "$retry" ] && [ ! -L "$retry" ] || return 1 + offset=$(inactive_pending_offset "$pending.offset") || return 1 + tags=$(mktemp "$retry.tags.XXXXXX") || return 1 + sorted=$(mktemp "$retry.sorted.XXXXXX") || { rm -f "$tags"; return 1; } + tmp=$(mktemp "$retry.filtered.XXXXXX") || { + rm -f "$tags" "$sorted" + return 1 + } + [ -f "$tags" ] && [ ! -L "$tags" ] || { rm -f "$tags" "$sorted" "$tmp"; return 1; } + [ -f "$sorted" ] && [ ! -L "$sorted" ] || { rm -f "$tags" "$sorted" "$tmp"; return 1; } + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tags" "$sorted" "$tmp"; return 1; } + remaining=$(budget_remaining_secs "$deadline_ms") + [ "$remaining" -gt 0 ] || { rm -f "$tags" "$sorted" "$tmp"; return 124; } + if run_bounded_child "$remaining" perl - "$pending" "$offset" "$retry" <<'PERL' | fm_nofollow_write "$tags" +use strict; +use warnings; + +my ($pending, $offset, $retry) = @ARGV; +sub emit_file { + my ($path, $skip) = @_; + open(my $fh, '<', $path) or exit 1; + binmode($fh); + local $/ = "\0"; + while (defined(my $value = <$fh>)) { + if ($skip > 0) { + $skip--; + next; + } + $value =~ s/\0\z// or exit 1; + print unpack('H*', $value), "\tP\n" or exit 1; + } + close($fh) or exit 1; +} +emit_file($pending, $offset + 0); +open(my $rfh, '<', $retry) or exit 1; +binmode($rfh); +local $/ = "\0"; +while (defined(my $value = <$rfh>)) { + $value =~ s/\0\z// or exit 1; + print unpack('H*', $value), "\tR\n" or exit 1; +} +close($rfh) or exit 1; +PERL + then : + else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$tags" "$sorted" "$tmp" + return "$rc" + fi + remaining=$(budget_remaining_secs "$deadline_ms") + [ "$remaining" -gt 0 ] || { rm -f "$tags" "$sorted" "$tmp"; return 124; } + tab=$(printf '\t') + if run_bounded_child "$remaining" env LC_ALL=C sort -t "$tab" -k1,1 -k2,2 "$tags" | fm_nofollow_write "$sorted"; then :; else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$tags" "$sorted" "$tmp" + return "$rc" + fi + remaining=$(budget_remaining_secs "$deadline_ms") + [ "$remaining" -gt 0 ] || { rm -f "$tags" "$sorted" "$tmp"; return 124; } + if run_bounded_child "$remaining" perl - "$sorted" <<'PERL' | fm_nofollow_write "$tmp" +use strict; +use warnings; + +my $path = shift @ARGV; +open(my $fh, '<', $path) or exit 1; +my ($last, $has_pending, $has_retry, $value) = ('', 0, 0, ''); +sub flush_group { + return unless $last ne '' && $has_retry && !$has_pending; + print $value, "\0" or exit 1; +} +while (defined(my $line = <$fh>)) { + chomp $line; + my ($key, $kind) = split(/\t/, $line, 2); + exit 1 unless defined($key) && defined($kind) && $key =~ /\A(?:[0-9A-Fa-f]{2})*\z/ && $kind =~ /\A[PR]\z/; + if ($key ne $last) { + flush_group(); + $last = $key; + $has_pending = 0; + $has_retry = 0; + $value = pack('H*', $key); + } + $has_pending = 1 if $kind eq 'P'; + $has_retry = 1 if $kind eq 'R'; +} +close($fh) or exit 1; +flush_group(); +PERL + then : + else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$tags" "$sorted" "$tmp" + return "$rc" + fi + [ ! -L "$retry" ] || { rm -f "$tags" "$sorted" "$tmp"; return 1; } + mv -f "$tmp" "$retry" || { rm -f "$tmp"; return 1; } + rm -f "$tags" "$sorted" || return 1 +} + +inactive_pending_merge_retry() { + local pending=$1 retry=$2 deadline_ms=${3:-} rc path + inactive_pending_retry_exists "$retry" || return 0 + [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 + for path in "$retry" "${retry}.incoming."*; do + [ -e "$path" ] || [ -L "$path" ] || continue + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + inactive_pending_retry_filter_duplicates "$pending" "$path" "$deadline_ms" + rc=$? + [ "$rc" = 0 ] || return "$rc" + inactive_append_nul_records_atomic "$path" "$pending" "$deadline_ms" + rc=$? + [ "$rc" = 0 ] || return "$rc" + done +} + +inactive_persist_nul_suffix() { + local source=$1 target=$2 skip=$3 retry_source=${4:-} deadline_ms=${5:-} + [ -f "$source" ] && [ ! -L "$source" ] || return 1 + [ ! -L "$target" ] || return 1 + case "$skip" in ''|*[!0-9]*) return 1 ;; esac + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + if [ "$source" != "$target" ]; then + [ ! -e "$target" ] || return 1 + [ ! -L "$target" ] || return 1 + mv -f "$source" "$target" || return 1 + fi + if [ -n "$retry_source" ]; then + inactive_append_nul_records_atomic "$retry_source" "$target" "$deadline_ms" + local retry_rc=$? + if [ "$retry_rc" -ne 0 ]; then + if [ ! -e "$MERGE_TXN" ] && [ ! -L "$MERGE_TXN" ] \ + && inactive_pending_retry_defer "$retry_source" "${target}.retry" "$deadline_ms" 1; then + return 124 + fi + return "$retry_rc" + fi + fi + inactive_pending_offset_write "${target}.offset" "$skip" || return 1 +} + +inactive_persist_nul_prefix() { + local source=$1 target=$2 deadline_ms=${3:-} tmp + [ ! -L "$target" ] || return 1 + tmp=$(mktemp "$target.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + if ! inactive_copy_nul_prefix "$source" "$tmp" "$deadline_ms"; then + rm -f "$tmp" + return 1 + fi + [ ! -L "$target" ] || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$target" || { rm -f "$tmp"; return 1; } +} + +RECONCILE_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_SECS:-900}" 900 60 1800) +SCAN_BUDGET_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_BUDGET_SECS:-10}" 10 1 300) +REPORTED_ROUTE_REPAIR_LIMIT=$(bounded_secs "${FM_REPORTED_ROUTE_REPAIR_LIMIT:-32}" 32 1 256) +PENDING_RECEIPT_REPUBLISH_LIMIT=$(bounded_secs "${FM_PENDING_RECEIPT_REPUBLISH_LIMIT:-32}" 32 1 256) +MAINTENANCE_TURN_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_MAINTENANCE_RESERVE_SECS:-1}" 1 1 60) +DIRECT_SCAN_BUDGET_SECS=$SCAN_BUDGET_SECS + +meta_value() { # + awk -F= -v wanted="$2" '$1 == wanted { print substr($0, index($0, "=") + 1); exit }' "$1" 2>/dev/null +} + +meta_value_unique() { # + awk -F= -v wanted="$2" ' + $1 == wanted { count++; value=substr($0, index($0, "=") + 1) } + END { + if (count == 1) { print value; exit 0 } + if (count == 0) exit 1 + exit 2 + } + ' "$1" 2>/dev/null +} + +file_mtime() { + if [ "$(uname -s 2>/dev/null)" = Darwin ]; then + stat -f '%m' "$1" 2>/dev/null + else + stat -c '%Y' "$1" 2>/dev/null + fi +} + +latest_activity() { + local id=$1 meta="$STATE/$1.meta" path m latest=0 + for path in "$meta" "$STATE/$id.status" "$STATE/$id.turn-ended"; do + if [ ! -e "$path" ] || [ -L "$path" ]; then + continue + fi + m=$(file_mtime "$path") || continue + [ "$m" -gt "$latest" ] && latest=$m + done + printf '%s' "$latest" +} + +hash_text() { + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$1" | shasum -a 256 | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print $1}' + else + return 1 + fi +} + +replay_parent_corr() { + local corr=${FM_PENDING_ROUTE_CORR:-} + case "${KIND:-}" in + secondmate) + fm_pending_reply_parent_corr_valid "$corr" || return 1 + [ -n "$corr" ] || return 1 + printf '%s' "$corr" + ;; + ship|scout|'') [ -z "$corr" ] || return 1; printf '%s' '' ;; + *) return 1 ;; + esac +} + +terminal_outcome_fingerprint() { + local id=$1 incarnation=$2 outcome=$3 snapshot=$4 kind=$5 parent_corr=${6:-} source=${7:-} + case "$source" in + run-step|pane) ;; + *) return 1 ;; + esac + case "$kind" in + secondmate) + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + hash_text "$id|$incarnation|$outcome|$snapshot|$kind|$source|$parent_corr" + ;; + ship|scout) + [ -z "$parent_corr" ] || return 1 + hash_text "$id|$incarnation|$outcome|$snapshot|$kind|$source" + ;; + *) return 1 ;; + esac +} + +metadata_fingerprint() { + local meta=$1 contents + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + contents=$(LC_ALL=C cat "$meta") || return 1 + hash_text "$contents" +} + +scan_cursor_write() { + local id=$1 identity=$2 cursor_tmp identity_tmp + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$identity" in ''|*[!0123456789abcdefABCDEF]*) return 1 ;; esac + [ "${#identity}" = 64 ] || return 1 + [ ! -L "$SCAN_CURSOR" ] && [ ! -L "$SCAN_CURSOR_IDENTITY" ] || return 1 + identity_tmp=$(mktemp "$SCAN_CURSOR_IDENTITY.XXXXXX") || return 1 + cursor_tmp=$(mktemp "$SCAN_CURSOR.XXXXXX") || { rm -f "$identity_tmp"; return 1; } + [ -f "$identity_tmp" ] && [ ! -L "$identity_tmp" ] || { rm -f "$identity_tmp" "$cursor_tmp"; return 1; } + [ -f "$cursor_tmp" ] && [ ! -L "$cursor_tmp" ] || { rm -f "$identity_tmp" "$cursor_tmp"; return 1; } + if ! printf '%s\n' "$identity" | fm_nofollow_write "$identity_tmp" \ + || ! fm_nofollow_rename "$identity_tmp" "$SCAN_CURSOR_IDENTITY"; then + rm -f "$identity_tmp" "$cursor_tmp" + return 1 + fi + if ! printf '%s\n' "$id" | fm_nofollow_write "$cursor_tmp" \ + || ! fm_nofollow_rename "$cursor_tmp" "$SCAN_CURSOR"; then + rm -f "$cursor_tmp" + return 1 + fi +} + +single_line() { + printf '%s' "$1" | tr '\t\r\n' ' ' +} + +valid_task_id() { + printf '%s' "$1" | grep -Eq '^[A-Za-z0-9][A-Za-z0-9._-]*$' +} + +is_secondmate_home() { + local marker=$ROUTE_MARKER value + [ ! -L "$marker" ] || return 2 + if [ ! -e "$marker" ]; then + return 1 + fi + [ -f "$marker" ] || return 2 + value=$(cat "$marker" 2>/dev/null || true) + case "$value" in + ''|*[!A-Za-z0-9._-]*) return 2 ;; + esac + return 0 +} + +herdr_identity_allowed() { # + local meta=$1 backend session window rc + if backend=$(meta_value_unique "$meta" backend); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + backend=tmux + fi + case "$backend" in + tmux) return 0 ;; + herdr) ;; + *) return 1 ;; + esac + session=$(meta_value_unique "$meta" herdr_session) || return 1 + window=$(meta_value_unique "$meta" window) || return 1 + case "$session" in + firstmate) : ;; + default|DEFAULT|CAPTAIN|captain) return 1 ;; + *) return 1 ;; + esac + case "$window" in firstmate:*) return 0 ;; esac + return 1 +} + +run_bounded_child() { # [args...] + local seconds=$1 + shift + if [ "${FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT:-0}" != 1 ] \ + && command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif [ "${FM_INACTIVE_OUTCOME_FORCE_PORTABLE_TIMEOUT:-0}" != 1 ] \ + && command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + elif command -v perl >/dev/null 2>&1; then + perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV or exit 127 } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; my $status = $?; exit(($status & 127) ? 128 + ($status & 127) : ($status >> 8))' "$seconds" "$@" + else + return 125 + fi +} + +receipt_path() { # + printf '%s/%s.%s' "$OUTCOME_DIR" "$1" "$2" +} + +receipt_field() { # + meta_value_unique "$1" "$2" +} + +claim_path() { # + printf '%s/.%s.claim' "$OUTCOME_DIR" "$1" +} + +claim_publish() { + fm_nofollow_rename "$1" "$2" "${3:-0}" +} + +claim_field() { # + meta_value_unique "$1" "$2" +} + +claim_binary_field() { # + local value + value=$(claim_field "$1" "$2") || return 1 + case "$value" in + 0|1) printf '%s' "$value" ;; + *) return 1 ;; + esac +} + +claim_binary_fields_valid() { + local claim=$1 field value rc + for field in output_started output_emitted output_complete output_confirmed caller_confirmed; do + value=$(awk -F= -v wanted="$field" ' + $1 == wanted { count++; value=substr($0, index($0, "=") + 1) } + END { + if (count == 0) exit 1 + if (count != 1) exit 2 + print value + } + ' "$claim" 2>/dev/null) + rc=$? + case "$rc" in + 1) continue ;; + 0) + case "$value" in 0|1) ;; *) return 1 ;; esac + ;; + *) return 1 ;; + esac + done +} + +claim_receipt_state() { + local fp=$1 suffix path found= + for suffix in pending presented reported; do + path=$(receipt_path "$fp" "$suffix") + [ ! -L "$path" ] || return 2 + [ -e "$path" ] || continue + [ -f "$path" ] || return 2 + prepare_pending_receipt "$path" || return 2 + [ -z "$found" ] || return 2 + found=$suffix + done + [ -n "$found" ] || return 1 + printf '%s' "$found" +} + +claim_defer_generation_live() { + local generation=$1 stored_start=${2:-} + case "$generation" in ''|*[!0-9]*) return 1 ;; esac + [ "$generation" != 0 ] || return 1 + [ -n "$stored_start" ] || return 1 + kill -0 "$generation" 2>/dev/null || return 1 + fm_pid_start_matches_stored "$generation" "$stored_start" +} + +drain_claim_owner() { + local row=$1 owner parent_pid parent_start drain_file drain_dir state_dir drain_base + owner=$(fm_lock_link_owner "$FM_WAKE_QUEUE_LOCK" 2>/dev/null || true) + if [ "${FM_WAKE_DRAIN_DELEGATED:-0}" = 1 ]; then + parent_pid=${FM_WAKE_DRAIN_PARENT_PID:-} + else + parent_pid=${PPID:-} + fi + parent_start=${FM_WAKE_DRAIN_PARENT_START:-} + drain_file=${FM_WAKE_DRAIN_FILE:-} + [ -n "$owner" ] && [ -n "$parent_pid" ] && [ -n "$parent_start" ] && [ -n "$row" ] && [ -n "$drain_file" ] || return 1 + fm_pid_start_matches_stored "$parent_pid" "$parent_start" || return 1 + [ "$(cat "$owner/pid" 2>/dev/null || true)" = "$parent_pid" ] || return 1 + fm_lock_points_to_owner "$FM_WAKE_QUEUE_LOCK" "$owner" || return 1 + [ -f "$drain_file" ] && [ ! -L "$drain_file" ] || return 1 + drain_dir=$(cd "$(dirname "$drain_file")" 2>/dev/null && pwd -P) || return 1 + state_dir=$(cd "$STATE" 2>/dev/null && pwd -P) || return 1 + [ "$drain_dir" = "$state_dir" ] || return 1 + if [ "${FM_WAKE_DRAIN_RESUMED_SOURCE:-0}" = 1 ]; then + drain_base=$(basename "$drain_file") || return 1 + case "$drain_base" in + .wake-queue) ;; + .wake-queue.deduped.*|.wake-queue.drain.*) + case "${drain_base##*.}" in ''|*[!0-9]*) return 1 ;; esac + ;; + *) return 1 ;; + esac + else + [ "$(basename "$drain_file")" = ".wake-queue.deduped.$parent_pid" ] || return 1 + fi + awk -v wanted="$row" '$0 == wanted { found=1; exit } END { exit !found }' "$drain_file" +} + +claim_validate() { # + local claim=$1 fp=$2 row=$3 state + [ -f "$claim" ] && [ ! -L "$claim" ] || return 1 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 1 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 1 + [ "$(claim_field "$claim" row)" = "$row" ] || return 1 + state=$(claim_field "$claim" state) + case "$state" in reserved|presenting|presented) printf '%s' "$state" ;; *) return 1 ;; esac +} + +claim_validate_caller_owner() { # + local claim=$1 caller_pid=$2 caller_start + [ -f "$claim" ] && [ ! -L "$claim" ] || return 1 + case "$caller_pid" in ''|*[!0-9]*|0) return 1 ;; esac + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 1 + [ "$(claim_field "$claim" defer_generation 2>/dev/null || true)" = "$caller_pid" ] || return 1 + caller_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + fm_pid_start_matches_stored "$caller_pid" "$caller_start" +} + +claim_rewrite_row() { + local claim=$1 row=$2 tmp line + tmp=$(mktemp "$OUTCOME_DIR/.claim-row.XXXXXX") || return 1 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in row=*) printf 'row=%s\n' "$row" ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 1; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 1; } +} + +claim_recover_deferred_handoff() { + local key=$1 row=$2 generation=${FM_WAKE_DRAIN_GENERATION:-} generation_start + local fp claim state tmp line seen_generation=0 seen_generation_start=0 seen_caller_confirmed=0 + [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] || return 2 + case "$generation" in ''|*[!0-9]*|0) return 2 ;; esac + generation_start=$(fm_pid_start "$generation") || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 0 ] || return 2 + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presented\n' ;; + defer_generation=*) printf 'defer_generation=%s\n' "$generation"; seen_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=%s\n' "$generation_start"; seen_generation_start=1 ;; + caller_confirmed=*) printf 'caller_confirmed=1\n'; seen_caller_confirmed=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_generation" = 1 ] || printf 'defer_generation=%s\n' "$generation" + [ "$seen_generation_start" = 1 ] || printf 'defer_generation_start=%s\n' "$generation_start" + [ "$seen_caller_confirmed" = 1 ] || printf 'caller_confirmed=1\n' + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_rebind_deferred_generation() { + local key=$1 row=$2 generation=${FM_WAKE_DRAIN_GENERATION:-} generation_start + local fp claim state tmp line seen_generation=0 seen_generation_start=0 + case "$generation" in ''|*[!0-9]*|0) return 2 ;; esac + generation_start=$(fm_pid_start "$generation") || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + case "$state" in presenting|presented) ;; *) return 2 ;; esac + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + defer_generation=*) printf 'defer_generation=%s\n' "$generation"; seen_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=%s\n' "$generation_start"; seen_generation_start=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_generation" = 1 ] || printf 'defer_generation=%s\n' "$generation" + [ "$seen_generation_start" = 1 ] || printf 'defer_generation_start=%s\n' "$generation_start" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_recorded_presented() { # + local key=$1 row=$2 fp claim state tmp line field value + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + case "$state" in reserved|presenting) ;; *) return 2 ;; esac + claim_binary_fields_valid "$claim" || return 2 + for field in output_started output_emitted output_complete output_confirmed caller_confirmed; do + value=$(claim_binary_field "$claim" "$field" 2>/dev/null || true) + case "$value" in ''|0) ;; *) return 2 ;; esac + done + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presented\n' ;; + output_started=*) printf 'output_started=1\n' ;; + output_emitted=*) printf 'output_emitted=1\n' ;; + output_complete=*) printf 'output_complete=1\n' ;; + output_confirmed=*) printf 'output_confirmed=1\n' ;; + caller_confirmed=*) printf 'caller_confirmed=1\n' ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + for field in output_started output_emitted output_complete output_confirmed caller_confirmed; do + grep -Fq "$field=" "$claim" || printf '%s=1\n' "$field" + done + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_reserve() { # + local key=$1 row=$2 fp claim tmp state existing old_row line output_started output_emitted output_complete output_confirmed caller_confirmed defer_ack + local defer_generation defer_generation_start receipt_state receipt_rc=0 recorded_report=0 report_rc=1 recorded_ready=1 value + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + inactive_state_preflight || return 2 + receipt_state=$(claim_receipt_state "$fp" 2>/dev/null) || receipt_rc=$? + [ "$receipt_rc" = 0 ] || return 2 + case "$receipt_state" in + presented|reported) + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + if [ -e "$claim" ]; then + [ -f "$claim" ] || return 2 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 2 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 2 + [ -n "$(claim_field "$claim" row)" ] || return 2 + case "$(claim_field "$claim" state)" in reserved|presenting|presented) ;; *) return 2 ;; esac + rm -f "$claim" || return 2 + fi + return 3 + ;; + pending) ;; + *) return 2 ;; + esac + prepare_pending_receipt "$(receipt_path "$fp" pending)" || return 2 + report_rc=0 + pending_secondmate_report_recorded "$fp" >/dev/null || report_rc=$? + case "$report_rc" in + 0) recorded_report=1 ;; + 1) ;; + *) return 2 ;; + esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + if [ -e "$claim" ]; then + [ -f "$claim" ] || return 2 + [ "$(claim_field "$claim" schema)" = fm-inactive-outcome-claim.v1 ] || return 2 + [ "$(claim_field "$claim" fingerprint)" = "$fp" ] || return 2 + state=$(claim_field "$claim" state) + case "$state" in presented|presenting|reserved) ;; *) return 2 ;; esac + old_row=$(claim_field "$claim" row) + [ -n "$old_row" ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + if [ "$state" = presented ]; then + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + fi + fi + if [ "$state" = presenting ]; then + output_started=$(claim_field "$claim" output_started 2>/dev/null || true) + output_emitted=$(claim_field "$claim" output_emitted 2>/dev/null || true) + output_complete=$(claim_field "$claim" output_complete 2>/dev/null || true) + output_confirmed=$(claim_field "$claim" output_confirmed 2>/dev/null || true) + caller_confirmed=$(claim_field "$claim" caller_confirmed 2>/dev/null || true) + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + if [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ + && claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then + return 4 + fi + fi + fi + if [ "$old_row" != "$row" ]; then + claim_rewrite_row "$claim" "$row" || return 2 + fi + if [ "$state" = presented ] && [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ]; then + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + if [ -n "${FM_WAKE_DRAIN_GENERATION:-}" ] \ + && { [ "${FM_WAKE_DRAIN_DIRECT:-0}" = 1 ] \ + || ! claim_defer_generation_live "$defer_generation" "$defer_generation_start"; }; then + claim_rebind_deferred_generation "$key" "$row" || return 2 + fi + return 5 + fi + if [ "$state" = presenting ] && [ "$defer_ack" = 1 ]; then + if [ "$recorded_report" = 1 ]; then + recorded_ready=1 + for value in "$output_started" "$output_emitted" "$output_complete" "$output_confirmed" "$caller_confirmed"; do + case "$value" in ''|0) ;; *) recorded_ready=0 ;; esac + done + if [ "$recorded_ready" = 1 ]; then + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_recorded_presented "$key" "$row" || return 2 + return 5 + fi + fi + if [ "$caller_confirmed" = 0 ] \ + && [ "$output_started" = 1 ] \ + && [ "$output_emitted" = 1 ] \ + && [ "$output_complete" = 1 ] \ + && [ "$output_confirmed" = 1 ] \ + && [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] \ + && [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ + && ! claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then + claim_recover_deferred_handoff "$key" "$row" || return 2 + return 5 + fi + case "$caller_confirmed" in + 1) + [ "$output_started" = 1 ] || return 2 + [ "$output_emitted" = 1 ] || return 2 + [ "$output_complete" = 1 ] || return 2 + [ "$output_confirmed" = 1 ] || return 2 + if [ -n "${FM_WAKE_DRAIN_GENERATION:-}" ]; then + claim_rebind_deferred_generation "$key" "$row" || return 2 + fi + claim_mark_presented_recovered "$key" "$row" || return 2 + return 5 + ;; + ''|0) ;; + *) return 2 ;; + esac + case "$output_emitted" in + ''|0) ;; + 1) [ "$output_confirmed" = 1 ] || return 2 ;; + *) return 2 ;; + esac + case "$output_complete" in + ''|0) ;; + 1) return 4 ;; + *) return 2 ;; + esac + [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] || return 4 + return 0 + fi + if [ "$recorded_report" = 1 ]; then + if [ "$state" = presented ]; then + return 5 + fi + if [ "$state" = reserved ]; then + claim_mark_presenting "$key" "$row" || return 2 + state=presenting + output_started=0 + output_emitted=0 + output_complete=0 + output_confirmed=0 + caller_confirmed=0 + fi + for value in "$output_started" "$output_emitted" "$output_complete" "$output_confirmed" "$caller_confirmed"; do + case "$value" in ''|0) ;; *) recorded_ready=0 ;; esac + done + if [ "$state" = presenting ] && [ "$recorded_ready" = 1 ]; then + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_recorded_presented "$key" "$row" || return 2 + return 5 + fi + fi + if [ "$recorded_report" = 1 ]; then + case "$state" in + presented) return 5 ;; + presenting|reserved) + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_output_complete "$key" "$row" || return 2 + claim_mark_output_confirmed "$key" "$row" || return 2 + claim_mark_presented "$key" "$row" || return 2 + return 5 + ;; + esac + fi + if [ "$state" = presenting ]; then + output_complete=$(claim_binary_field "$claim" output_complete 2>/dev/null || true) + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null || true) + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null || true) + if [ "${FM_WAKE_DRAIN_DIRECT:-0}" != 1 ] \ + && claim_defer_generation_live "$defer_generation" "$defer_generation_start"; then + return 4 + fi + fi + if [ "$output_complete" = 1 ]; then + [ "$output_confirmed" = 1 ] || return 4 + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi + if [ "$output_emitted" = 1 ]; then + [ "$output_confirmed" = 1 ] || return 4 + if claim_mark_output_complete "$key" "$row"; then + claim_mark_presented "$key" "$row" || return 2 + return 5 + fi + return 6 + fi + if [ "$defer_ack" = 1 ]; then + return 0 + fi + fi + if [ "$state" = presented ]; then + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null || true) + if [ "$defer_ack" = 1 ]; then + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=reserved\n' ;; + defer_ack=*) printf 'defer_ack=0\n' ;; + defer_generation=*) printf 'defer_generation=\n' ;; + defer_generation_start=*) printf 'defer_generation_start=\n' ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } + return 0 + fi + return 5 + fi + return 0 + fi + mkdir -p "$OUTCOME_DIR" || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + printf 'schema=fm-inactive-outcome-claim.v1\n' + printf 'fingerprint=%s\n' "$fp" + printf 'row=%s\n' "$row" + printf 'state=reserved\n' + printf 'created_epoch=%s\n' "$(date +%s)" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + if claim_publish "$tmp" "$claim" 1; then + rm -f "$tmp" + if [ "$recorded_report" = 1 ]; then + claim_mark_presenting "$key" "$row" || return 2 + claim_mark_recorded_presented "$key" "$row" || return 2 + return 5 + fi + return 0 + fi + rm -f "$tmp" + [ -e "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presented ] && return 1 + return 0 +} + +claim_mark_presenting() { # + local key=$1 row=$2 fp claim state tmp line defer_ack=0 defer_generation='' defer_generation_start='' + local preserve_output=0 seen_pid=0 seen_output=0 seen_emitted=0 seen_complete=0 seen_caller_confirmed=0 + local seen_defer_ack=0 seen_defer_generation=0 seen_defer_generation_start=0 + if [ "${FM_WAKE_DRAIN_DEFER_ACK:-0}" = 1 ] || [ -n "${FM_WAKE_DRAIN_GENERATION:-}" ]; then + defer_ack=1 + fi + if [ "$defer_ack" = 1 ]; then + defer_generation=${FM_WAKE_DRAIN_GENERATION:-} + case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac + defer_generation_start=$(fm_pid_start "$defer_generation") || return 2 + fi + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + case "$state" in + presenting|reserved) ;; + *) return 2 ;; + esac + claim_binary_fields_valid "$claim" || return 2 + if [ "$state" = presenting ] \ + && [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ]; then + preserve_output=1 + fi + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presenting\n' ;; + presentation_pid=*) printf 'presentation_pid=%s\n' "${BASHPID:-$$}"; seen_pid=1 ;; + output_started=*) + if [ "$preserve_output" = 1 ]; then printf '%s\n' "$line"; else printf 'output_started=0\n'; fi + seen_output=1 + ;; + output_emitted=*) + if [ "$preserve_output" = 1 ]; then printf '%s\n' "$line"; else printf 'output_emitted=0\n'; fi + seen_emitted=1 + ;; + output_complete=*) + if [ "$preserve_output" = 1 ]; then printf '%s\n' "$line"; else printf 'output_complete=0\n'; fi + seen_complete=1 + ;; + caller_confirmed=*) + if [ "$preserve_output" = 1 ]; then printf '%s\n' "$line"; else printf 'caller_confirmed=0\n'; fi + seen_caller_confirmed=1 + ;; + defer_ack=*) printf 'defer_ack=%s\n' "$defer_ack"; seen_defer_ack=1 ;; + defer_generation=*) printf 'defer_generation=%s\n' "$defer_generation"; seen_defer_generation=1 ;; + defer_generation_start=*) printf 'defer_generation_start=%s\n' "$defer_generation_start"; seen_defer_generation_start=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_pid" = 1 ] || printf 'presentation_pid=%s\n' "${BASHPID:-$$}" + [ "$seen_output" = 1 ] || printf 'output_started=0\n' + [ "$seen_emitted" = 1 ] || printf 'output_emitted=0\n' + [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' + [ "$seen_caller_confirmed" = 1 ] || printf 'caller_confirmed=0\n' + [ "$seen_defer_ack" = 1 ] || printf 'defer_ack=%s\n' "$defer_ack" + [ "$seen_defer_generation" = 1 ] || printf 'defer_generation=%s\n' "$defer_generation" + [ "$seen_defer_generation_start" = 1 ] || printf 'defer_generation_start=%s\n' "$defer_generation_start" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_output_complete() { # + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} + local fp claim state tmp line seen_output=0 seen_complete=0 + local seen_emitted=0 seen_confirmed=0 seen_caller_confirmed=0 confirmed_value caller_confirmed_value + case "$owner_required" in + 1) drain_claim_owner "$row" || return 2 ;; + 0) + case "$expected_generation" in ''|*[!0-9]*|0) return 2 ;; esac + ;; + *) return 2 ;; + esac + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + if [ "$owner_required" = 0 ]; then + claim_validate_caller_owner "$claim" "$expected_generation" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + fi + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; + output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; + output_confirmed=*) + [ "$seen_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + confirmed_value=${line#output_confirmed=} + case "$confirmed_value" in 0|1) ;; *) rm -f "$tmp"; return 2 ;; esac + [ "$owner_required" = 0 ] && confirmed_value=1 + printf 'output_confirmed=%s\n' "$confirmed_value" + seen_confirmed=1 + ;; + caller_confirmed=*) + [ "$seen_caller_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + caller_confirmed_value=${line#caller_confirmed=} + case "$caller_confirmed_value" in 0|1) ;; *) rm -f "$tmp"; return 2 ;; esac + [ "$owner_required" = 0 ] && caller_confirmed_value=1 + printf 'caller_confirmed=%s\n' "$caller_confirmed_value" + seen_caller_confirmed=1 + ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_output" = 1 ] || printf 'output_started=1\n' + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' + [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' + [ "$seen_confirmed" = 1 ] || \ + printf 'output_confirmed=%s\n' "$([ "$owner_required" = 0 ] && printf 1 || printf 0)" + [ "$seen_caller_confirmed" = 1 ] || \ + printf 'caller_confirmed=%s\n' "$([ "$owner_required" = 0 ] && printf 1 || printf 0)" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_output_started() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_complete=0 + local seen_emitted=0 seen_confirmed=0 seen_caller_confirmed=0 + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=0\n'; seen_emitted=1 ;; + output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; + output_confirmed=*) + [ "$seen_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'output_confirmed=0\n' + seen_confirmed=1 + ;; + caller_confirmed=*) + [ "$seen_caller_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'caller_confirmed=0\n' + seen_caller_confirmed=1 + ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_output" = 1 ] || printf 'output_started=1\n' + [ "$seen_emitted" = 1 ] || printf 'output_emitted=0\n' + [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' + [ "$seen_confirmed" = 1 ] || printf 'output_confirmed=0\n' + [ "$seen_caller_confirmed" = 1 ] || printf 'caller_confirmed=0\n' + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_output_emitted() { # + local key=$1 row=$2 fp claim state tmp line seen_output=0 seen_emitted=0 seen_complete=0 + local seen_confirmed=0 seen_caller_confirmed=0 + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; + output_complete=*) printf 'output_complete=0\n'; seen_complete=1 ;; + output_confirmed=*) + [ "$seen_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'output_confirmed=0\n' + seen_confirmed=1 + ;; + caller_confirmed=*) + [ "$seen_caller_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'caller_confirmed=0\n' + seen_caller_confirmed=1 + ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_output" = 1 ] || printf 'output_started=1\n' + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' + [ "$seen_complete" = 1 ] || printf 'output_complete=0\n' + [ "$seen_confirmed" = 1 ] || printf 'output_confirmed=0\n' + [ "$seen_caller_confirmed" = 1 ] || printf 'caller_confirmed=0\n' + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_output_emitted() { + local key=$1 row=$2 fp claim state emitted + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + emitted=$(claim_binary_field "$claim" output_emitted 2>/dev/null || true) + case "$emitted" in + 1) return 0 ;; + 0|'') return 1 ;; + *) return 2 ;; + esac +} + +claim_mark_output_confirmed() { # + local key=$1 row=$2 fp claim state tmp line seen_confirmed=0 seen_caller_confirmed=0 + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + output_confirmed=*) + [ "$seen_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'output_confirmed=1\n' + seen_confirmed=1 + ;; + caller_confirmed=*) + [ "$seen_caller_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'caller_confirmed=0\n' + seen_caller_confirmed=1 + ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_confirmed" = 1 ] || printf 'output_confirmed=1\n' + [ "$seen_caller_confirmed" = 1 ] || printf 'caller_confirmed=0\n' + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_presented() { # + local key=$1 row=$2 fp claim state tmp line defer_ack defer_generation defer_generation_start + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + defer_ack=$(claim_field "$claim" defer_ack 2>/dev/null) || return 2 + case "$defer_ack" in 0) ;; 1) + defer_generation=$(claim_field "$claim" defer_generation 2>/dev/null) || return 2 + defer_generation_start=$(claim_field "$claim" defer_generation_start 2>/dev/null) || return 2 + case "$defer_generation" in ''|*[!0-9]*|0) return 2 ;; esac + [ -n "$defer_generation_start" ] || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + ;; + *) return 2 ;; + esac + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presented\n' ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_presented_recovered() { # + local key=$1 row=$2 fp claim state tmp line + drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + [ "$state" = presenting ] || return 2 + claim_binary_fields_valid "$claim" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in state=*) printf 'state=presented\n' ;; *) printf '%s\n' "$line" ;; esac + done < "$claim" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_mark_confirmed() { # + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} + local fp claim state tmp line seen_output=0 seen_emitted=0 seen_complete=0 seen_confirmed=0 seen_caller_confirmed=0 + case "$owner_required" in + 1) drain_claim_owner "$row" || return 2 ;; + 0) ;; + *) return 2 ;; + esac + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + [ ! -L "$claim" ] || return 2 + state=$(claim_validate "$claim" "$fp" "$row") || return 2 + claim_binary_fields_valid "$claim" || return 2 + if [ "$owner_required" = 0 ]; then + claim_validate_caller_owner "$claim" "$expected_generation" || return 2 + [ "$(claim_binary_field "$claim" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + fi + case "$state" in + presented) + [ "$(claim_binary_field "$claim" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$claim" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + if [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ]; then + [ "$(claim_binary_field "$claim" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + fi + return 0 + ;; + presenting) ;; + *) return 2 ;; + esac + [ "$(claim_field "$claim" defer_ack 2>/dev/null || true)" = 1 ] || return 2 + tmp=$(mktemp "$OUTCOME_DIR/.claim-state.XXXXXX") || return 2 + chmod 600 "$tmp" 2>/dev/null || true + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + state=*) printf 'state=presented\n' ;; + output_started=*) printf 'output_started=1\n'; seen_output=1 ;; + output_emitted=*) printf 'output_emitted=1\n'; seen_emitted=1 ;; + output_complete=*) printf 'output_complete=1\n'; seen_complete=1 ;; + output_confirmed=*) + [ "$seen_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'output_confirmed=1\n' + seen_confirmed=1 + ;; + caller_confirmed=*) + [ "$seen_caller_confirmed" = 0 ] || { rm -f "$tmp"; return 2; } + [ "$owner_required" = 0 ] || { rm -f "$tmp"; return 2; } + printf 'caller_confirmed=1\n' + seen_caller_confirmed=1 + ;; + *) printf '%s\n' "$line" ;; + esac + done < "$claim" + [ "$seen_output" = 1 ] || printf 'output_started=1\n' + [ "$seen_emitted" = 1 ] || printf 'output_emitted=1\n' + [ "$seen_complete" = 1 ] || printf 'output_complete=1\n' + [ "$seen_confirmed" = 1 ] || printf 'output_confirmed=1\n' + [ "$owner_required" = 0 ] && [ "$seen_caller_confirmed" = 1 ] || { rm -f "$tmp"; return 2; } + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 2; } + [ ! -L "$claim" ] || { rm -f "$tmp"; return 2; } + claim_publish "$tmp" "$claim" || { rm -f "$tmp"; return 2; } +} + +claim_remove() { # + local key=$1 row=$2 owner_required=${3:-1} expected_generation=${4:-} fp claim + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 2 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + claim=$(claim_path "$fp") + case "$owner_required" in + 0) claim_validate_caller_owner "$claim" "$expected_generation" || return 2 ;; + 1) drain_claim_owner "$row" || return 2 ;; + *) return 2 ;; + esac + [ ! -L "$claim" ] || return 2 + [ -e "$claim" ] || return 0 + claim_validate "$claim" "$fp" "$row" >/dev/null || return 2 + rm -f "$claim" +} + +receipt_matches_expected() { + local existing=$1 + [ ! -L "$existing" ] || return 1 + [ -f "$existing" ] || return 1 + [ "$(receipt_field "$existing" schema)" = fm-jt-terminal-outcome.v1 ] || return 1 + [ "$(receipt_field "$existing" fingerprint)" = "$FP" ] || return 1 + [ "$(receipt_field "$existing" task_id)" = "$ID" ] || return 1 + [ "$(receipt_field "$existing" incarnation)" = "$INC" ] || return 1 + [ "$(receipt_field "$existing" outcome)" = "$OUTCOME" ] || return 1 + [ "$(receipt_field "$existing" terminal_source)" = "$SOURCE" ] || return 1 + [ "$(receipt_field "$existing" terminal_snapshot)" = "$SNAPSHOT" ] || return 1 + [ "$(receipt_field "$existing" kind)" = "$KIND" ] || return 1 + if [ "$KIND" = secondmate ]; then + [ "$(receipt_field "$existing" parent_task_id)" = "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_home)" = "${FM_PENDING_ROUTE_PARENT_HOME:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_status)" = "${FM_PENDING_ROUTE_PARENT_STATUS:-}" ] || return 1 + [ "$(receipt_field "$existing" parent_corr)" = "${FM_PENDING_ROUTE_CORR:-}" ] || return 1 + else + [ -z "$(receipt_field "$existing" parent_task_id)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_home)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_status)" ] || return 1 + [ -z "$(receipt_field "$existing" parent_corr)" ] || return 1 + fi +} + +receipt_write() { # globals: FP ID INC OUTCOME SNAPSHOT KIND SOURCE + local pending tmp existing + inactive_state_preflight || return 1 + pending=$(receipt_path "$FP" pending) + for suffix in pending presented reported; do + existing=$(receipt_path "$FP" "$suffix") + [ ! -L "$existing" ] || return 1 + [ -e "$existing" ] || continue + receipt_matches_expected "$existing" || return 1 + return 0 + done + mkdir -p "$OUTCOME_DIR" || return 1 + tmp=$(mktemp "$OUTCOME_DIR/.receipt.XXXXXX") || return 1 + chmod 600 "$tmp" 2>/dev/null || true + { + printf 'schema=fm-jt-terminal-outcome.v1\n' + printf 'fingerprint=%s\n' "$FP" + printf 'task_id=%s\n' "$ID" + printf 'incarnation=%s\n' "$INC" + printf 'outcome=%s\n' "$OUTCOME" + printf 'terminal_source=%s\n' "$SOURCE" + printf 'terminal_snapshot=%s\n' "$SNAPSHOT" + printf 'kind=%s\n' "$KIND" + printf 'parent_task_id=%s\n' "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" + printf 'parent_home=%s\n' "${FM_PENDING_ROUTE_PARENT_HOME:-}" + printf 'parent_status=%s\n' "${FM_PENDING_ROUTE_PARENT_STATUS:-}" + printf 'parent_corr=%s\n' "${FM_PENDING_ROUTE_CORR:-}" + printf 'created_epoch=%s\n' "$(date +%s)" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + if fm_nofollow_rename "$tmp" "$pending" 1; then + rm -f "$tmp" + else + rm -f "$tmp" + receipt_matches_expected "$pending" || return 1 + fi + return 0 +} + +replay_pane_idle_publication_valid() { + local meta=$1 id=$2 window=$3 backend=$4 incarnation=$5 source=${6:-} + local deadline_ms=${FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS:-} now_ms remaining_ms capture_secs + if [ "$source" = run-step ]; then + [ "${SOURCE:-}" = run-step ] || return 1 + run_step_incarnation_evidence_valid "$id" "$incarnation" "${OUTCOME:-}" "${SNAPSHOT:-}" + return $? + fi + if [ "${FM_INACTIVE_OUTCOME_CHILD_BOUND:-0}" = 1 ]; then + fm_pane_idle_proof_valid "$STATE" "$meta" "$id" "$window" "$backend" "$incarnation" "$RECONCILE_SECS" + return $? + fi + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + if [ -n "$deadline_ms" ]; then + now_ms=$(clock_millis) + remaining_ms=$((deadline_ms - now_ms)) + [ "$remaining_ms" -gt 0 ] || return 1 + capture_secs=$(((remaining_ms + 999) / 1000)) + else + capture_secs=${FM_INACTIVE_OUTCOME_SCAN_REMAINING_SECS:-${FM_LOCK_WAIT_SECS:-1}} + fi + capture_secs=$(bounded_secs "$capture_secs" 1 1 300) + run_bounded_child "$capture_secs" "$SCRIPT_DIR/fm-inactive-reconcile.sh" _pane-idle \ + "$STATE" "$meta" "$id" "$window" "$backend" "$incarnation" "$RECONCILE_SECS" +} + +publish_receipt_and_wake() { + local status=0 release_lock=0 + local pane_meta=${1:-} pane_id=${2:-} pane_window=${3:-} pane_backend=${4:-} pane_incarnation=${5:-} source=${6:-} + FM_WAKE_APPEND_CREATED=0 + if [ "$WAKE_QUEUE_LOCK_HELD" != 1 ]; then + wake_queue_lock_acquire || return 75 + release_lock=1 + fi + if [ -n "$pane_meta" ] \ + && ! replay_pane_idle_publication_valid "$pane_meta" "$pane_id" "$pane_window" "$pane_backend" "$pane_incarnation" "$source"; then + [ "$release_lock" = 1 ] && wake_queue_lock_release || true + return 75 + fi + if receipt_write; then + if [ -f "$(receipt_path "$FP" pending)" ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + status=$? + fi + if [ "$release_lock" = 1 ]; then + wake_queue_lock_release || status=1 + fi + return "$status" +} + +receipt_existing_core() { + local suffix existing expected_fp existing_kind existing_fingerprint existing_task_id existing_incarnation + local existing_outcome existing_snapshot existing_source parent_id parent_home parent_status parent_corr current_corr + RECEIPT_EXISTING_SUFFIX= + current_corr=$(replay_parent_corr) || return 1 + expected_fp=$(terminal_outcome_fingerprint "$ID" "$INC" "$OUTCOME" "$SNAPSHOT" "$KIND" "$current_corr" "$SOURCE") || return 1 + [ "$expected_fp" = "$FP" ] || return 1 + for suffix in pending presented reported; do + existing=$(receipt_path "$FP" "$suffix") + [ ! -L "$existing" ] || return 2 + [ -e "$existing" ] || continue + [ -f "$existing" ] || return 2 + [ "$(receipt_field "$existing" schema)" = fm-jt-terminal-outcome.v1 ] || return 2 + existing_fingerprint=$(receipt_field "$existing" fingerprint) || return 2 + existing_task_id=$(receipt_field "$existing" task_id) || return 2 + existing_incarnation=$(receipt_field "$existing" incarnation) || return 2 + existing_outcome=$(receipt_field "$existing" outcome) || return 2 + existing_source=$(receipt_field "$existing" terminal_source) || return 2 + existing_snapshot=$(receipt_field "$existing" terminal_snapshot) || return 2 + existing_kind=$(receipt_field "$existing" kind) || return 2 + [ "$existing_fingerprint" = "$FP" ] || return 2 + [ "$existing_task_id" = "$ID" ] || return 2 + [ "$existing_incarnation" = "$INC" ] || return 2 + [ "$existing_outcome" = "$OUTCOME" ] || return 2 + [ "$existing_source" = "$SOURCE" ] || return 2 + [ "$existing_snapshot" = "$SNAPSHOT" ] || return 2 + [ "$existing_kind" = "$KIND" ] || return 2 + case "$existing_kind" in + ship|scout) + parent_id=$(receipt_field "$existing" parent_task_id) || return 2 + parent_home=$(receipt_field "$existing" parent_home) || return 2 + parent_status=$(receipt_field "$existing" parent_status) || return 2 + parent_corr=$(receipt_field "$existing" parent_corr) || return 2 + [ -z "$parent_id" ] && [ -z "$parent_home" ] && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 2 + ;; + secondmate) + parent_id=$(receipt_field "$existing" parent_task_id) || return 2 + parent_home=$(receipt_field "$existing" parent_home) || return 2 + parent_status=$(receipt_field "$existing" parent_status) || return 2 + parent_corr=$(receipt_field "$existing" parent_corr) || return 2 + [ "$parent_id" = "${FM_PENDING_ROUTE_SECOND_MATE_ID:-}" ] || return 2 + [ "$parent_home" = "${FM_PENDING_ROUTE_PARENT_HOME:-}" ] || return 2 + [ "$parent_status" = "${FM_PENDING_ROUTE_PARENT_STATUS:-}" ] || return 2 + [ "$parent_corr" = "${FM_PENDING_ROUTE_CORR:-}" ] || return 2 + case "$parent_home" in /*) ;; *) return 2 ;; esac + case "$parent_status" in /*) ;; *) return 2 ;; esac + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 2 + ;; + *) return 2 ;; + esac + expected_fp=$(terminal_outcome_fingerprint "$existing_task_id" "$existing_incarnation" \ + "$existing_outcome" "$existing_snapshot" "$existing_kind" "$parent_corr" "$existing_source") || return 2 + [ "$expected_fp" = "$existing_fingerprint" ] || return 2 + RECEIPT_EXISTING_SUFFIX=$suffix + return 0 + done + return 1 +} + +republish_existing_receipt_wake() { + local existing task outcome status=0 existing_rc release_lock=0 + local pane_meta=${1:-} pane_id=${2:-} pane_window=${3:-} pane_backend=${4:-} pane_incarnation=${5:-} source=${6:-} + FM_WAKE_APPEND_CREATED=0 + if [ "$WAKE_QUEUE_LOCK_HELD" != 1 ]; then + wake_queue_lock_acquire || return 75 + release_lock=1 + fi + if receipt_existing_core; then + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + existing=$(receipt_path "$FP" pending) + task=$(receipt_field "$existing" task_id) + outcome=$(receipt_field "$existing" outcome) + if [ -n "$pane_meta" ] \ + && ! replay_pane_idle_publication_valid "$pane_meta" "$pane_id" "$pane_window" "$pane_backend" "$pane_incarnation" "$source"; then + status=75 + else + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$task state=$outcome fingerprint=$FP" || status=$? + fi + fi + else + existing_rc=$? + [ "$existing_rc" = 1 ] || status=1 + fi + if [ "$release_lock" = 1 ]; then + wake_queue_lock_release || status=1 + fi + return "$status" +} + +publish_secondmate_receipt_and_wake() { + local route_lock status=0 route_invalid=0 existing_rc pending pending_corr pending_parent_id pending_parent_home pending_parent_status + local release_lock=0 + local pane_meta=${1:-} pane_id=${2:-} pane_window=${3:-} pane_backend=${4:-} pane_incarnation=${5:-} source=${6:-} + FM_WAKE_APPEND_CREATED=0 + if [ "$WAKE_QUEUE_LOCK_HELD" != 1 ]; then + wake_queue_lock_acquire || return 75 + release_lock=1 + fi + if [ -L "$FM_HOME" ] || [ ! -d "$FM_HOME" ] || [ -L "$STATE" ] || [ ! -d "$STATE" ]; then + [ "$release_lock" = 1 ] && wake_queue_lock_release || true + return 75 + fi + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$FM_HOME") + if ! fm_lock_acquire_wait "$route_lock"; then + [ "$release_lock" = 1 ] && wake_queue_lock_release || true + return 75 + fi + KIND=secondmate + pending=$(receipt_path "$FP" pending) + if [ -e "$pending" ] || [ -L "$pending" ]; then + [ -f "$pending" ] && [ ! -L "$pending" ] || route_invalid=1 + if [ "$route_invalid" = 0 ]; then + pending_corr=$(receipt_field "$pending" parent_corr) + printf '%s' "$pending_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || route_invalid=1 + fi + if [ "$route_invalid" = 0 ] && ! fm_pending_reply_secondmate_route_validate "$FM_HOME" "$pending_corr"; then + route_invalid=1 + fi + if [ "$route_invalid" = 0 ]; then + pending_parent_id=$(receipt_field "$pending" parent_task_id) + pending_parent_home=$(receipt_field "$pending" parent_home) + pending_parent_status=$(receipt_field "$pending" parent_status) + [ "$pending_parent_id" = "$FM_PENDING_ROUTE_SECOND_MATE_ID" ] || route_invalid=1 + [ "$pending_parent_home" = "$FM_PENDING_ROUTE_PARENT_HOME" ] || route_invalid=1 + [ "$pending_parent_status" = "$FM_PENDING_ROUTE_PARENT_STATUS" ] || route_invalid=1 + fi + elif ! fm_pending_reply_secondmate_route_validate "$FM_HOME"; then + route_invalid=1 + fi + if [ "$route_invalid" = 1 ]; then + fm_lock_release "$route_lock" || true + [ "$release_lock" = 1 ] && wake_queue_lock_release || true + return 75 + fi + if [ -n "$pane_meta" ] \ + && ! replay_pane_idle_publication_valid "$pane_meta" "$pane_id" "$pane_window" "$pane_backend" "$pane_incarnation" "$source"; then + fm_lock_release "$route_lock" || true + [ "$release_lock" = 1 ] && wake_queue_lock_release || true + return 75 + fi + if receipt_existing_core; then + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + existing_rc=$? + if [ "$existing_rc" = 1 ]; then + if receipt_write; then + if [ -f "$(receipt_path "$FP" pending)" ]; then + fm_wake_append_if_absent_locked FM_WAKE_APPEND_CREATED check "inactive-outcome:$FP" \ + "inactive terminal outcome: task=$ID state=$OUTCOME fingerprint=$FP" || status=$? + fi + else + status=$? + fi + else + status=1 + fi + fi + fm_lock_release "$route_lock" || status=1 + if [ "$release_lock" = 1 ]; then + wake_queue_lock_release || status=1 + fi + return "$status" +} + +prepare_pending_receipt() { + local pending=$1 expected_fp serialized_fp schema task_id incarnation outcome terminal_source terminal_snapshot kind key + local parent_task_id parent_home parent_status parent_corr + [ -f "$pending" ] && [ ! -L "$pending" ] || return 1 + FP=${pending##*/} + case "$FP" in + *.pending) FP=${FP%.pending} ;; + *.presented) FP=${FP%.presented} ;; + *.reported) FP=${FP%.reported} ;; + *) return 1 ;; + esac + case "$FP" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["fingerprint"]=1; allowed["task_id"]=1 + allowed["incarnation"]=1; allowed["outcome"]=1; allowed["terminal_source"]=1 + allowed["terminal_snapshot"]=1; allowed["kind"]=1; allowed["parent_task_id"]=1 + allowed["parent_home"]=1; allowed["parent_status"]=1; allowed["parent_corr"]=1 + allowed["created_epoch"]=1; valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + next + } + { valid=0 } + END { + for (key in allowed) { + if (key == "schema" || key == "fingerprint" || key == "task_id" || + key == "incarnation" || key == "outcome" || key == "terminal_source" || + key == "terminal_snapshot" || key == "kind") { + if (!(key in seen)) valid=0 + } + } + exit !valid + } + ' "$pending" 2>/dev/null || return 1 + schema=$(receipt_field "$pending" schema) || return 1 + task_id=$(receipt_field "$pending" task_id) || return 1 + incarnation=$(receipt_field "$pending" incarnation) || return 1 + outcome=$(receipt_field "$pending" outcome) || return 1 + terminal_source=$(receipt_field "$pending" terminal_source) || return 1 + terminal_snapshot=$(receipt_field "$pending" terminal_snapshot) || return 1 + kind=$(receipt_field "$pending" kind) || return 1 + serialized_fp=$(receipt_field "$pending" fingerprint) || return 1 + parent_task_id=$(receipt_field "$pending" parent_task_id 2>/dev/null || true) + parent_home=$(receipt_field "$pending" parent_home 2>/dev/null || true) + parent_status=$(receipt_field "$pending" parent_status 2>/dev/null || true) + parent_corr=$(receipt_field "$pending" parent_corr 2>/dev/null || true) + [ "$schema" = fm-jt-terminal-outcome.v1 ] || return 1 + [ -n "$task_id" ] && [ -n "$incarnation" ] && [ -n "$terminal_source" ] \ + && [ -n "$terminal_snapshot" ] || return 1 + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + case "$terminal_source" in + run-step|pane) ;; + *) return 1 ;; + esac + valid_task_id "$task_id" || return 1 + case "$outcome" in done|failed) ;; *) return 1 ;; esac + case "$kind" in ship|scout) + [ -z "$parent_task_id" ] && [ -z "$parent_home" ] \ + && [ -z "$parent_status" ] && [ -z "$parent_corr" ] || return 1 + ;; + secondmate) + [ -n "$parent_task_id" ] && [ -n "$parent_home" ] && [ -n "$parent_status" ] || return 1 + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + ;; + *) return 1 ;; + esac + expected_fp=$(terminal_outcome_fingerprint "$task_id" "$incarnation" "$outcome" \ + "$terminal_snapshot" "$kind" "$parent_corr" "$terminal_source") || return 1 + [ "$serialized_fp" = "$FP" ] || return 1 + [ "$expected_fp" = "$FP" ] || return 1 + if [ "$terminal_source" = run-step ] \ + && ! run_step_incarnation_evidence_valid "$task_id" "$incarnation" "$outcome" "$terminal_snapshot"; then + return 1 + fi + ID=$task_id + INC=$incarnation + OUTCOME=$outcome + SOURCE=$terminal_source + SNAPSHOT=$terminal_snapshot + KIND=$kind + return 0 +} + +parent_status_line_status() { + local parent_status=$1 wanted=$2 scan_secs now_ms deadline_ms status + [ -f "$parent_status" ] && [ ! -L "$parent_status" ] || return 2 + scan_secs=$(bounded_secs "${FM_INACTIVE_OUTCOME_PARENT_STATUS_SCAN_SECS:-1}" 1 1 60) + deadline_ms=${FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS:-} + case "$deadline_ms" in + ''|*[!0-9]*) ;; + *) + now_ms=$(clock_millis) + [ "$deadline_ms" -gt "$now_ms" ] || return 124 + [ "$((deadline_ms - now_ms))" -ge 1000 ] || return 124 + scan_secs=$(((deadline_ms - now_ms) / 1000)) + scan_secs=$(bounded_secs "$scan_secs" 1 1 60) + ;; + esac + run_bounded_child "$scan_secs" grep -Fqx -- "$wanted" "$parent_status" + status=$? + case "$status" in + 0|1|124) return "$status" ;; + *) return 2 ;; + esac +} + +parent_status_append_line() { + local parent_status=$1 line=$2 + command -v perl >/dev/null 2>&1 || return 1 + printf '%s\n' "$line" | perl -e ' + use Fcntl qw(:DEFAULT); + my ($parent_status) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + my $fh; + sysopen($fh, $parent_status, O_WRONLY | O_APPEND | O_CREAT | $nofollow, 0600) or exit 1; + binmode($fh); + local $/; + my $line = // ""; + print($fh $line) or exit 1; + close($fh) or exit 1; + ' "$parent_status" +} + +pending_secondmate_report_recorded() { + local fp=$1 pending kind task_id outcome parent_task_id parent_home parent_status parent_corr line status + local route_lock route_status=0 + pending=$(receipt_path "$fp" pending) + kind=$(receipt_field "$pending" kind) || return 2 + case "$kind" in + ship|scout) return 1 ;; + secondmate) ;; + *) return 2 ;; + esac + prepare_pending_receipt "$pending" || return 2 + task_id=$(receipt_field "$pending" task_id) || return 2 + outcome=$(receipt_field "$pending" outcome) || return 2 + parent_task_id=$(receipt_field "$pending" parent_task_id) || return 2 + parent_home=$(receipt_field "$pending" parent_home) || return 2 + parent_status=$(receipt_field "$pending" parent_status) || return 2 + parent_corr=$(receipt_field "$pending" parent_corr) || return 2 + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$FM_HOME") + fm_lock_acquire_wait "$route_lock" || return 2 + if ! fm_pending_reply_secondmate_route_validate "$FM_HOME" "$parent_corr" 2 \ + || [ "$FM_PENDING_ROUTE_SECOND_MATE_ID" != "$parent_task_id" ] \ + || [ "$FM_PENDING_ROUTE_PARENT_HOME" != "$parent_home" ] \ + || [ "$FM_PENDING_ROUTE_PARENT_STATUS" != "$parent_status" ] \ + || [ "$FM_PENDING_ROUTE_CORR" != "$parent_corr" ]; then + route_status=2 + fi + fm_lock_release "$route_lock" || route_status=2 + [ "$route_status" = 0 ] || return "$route_status" + [ -f "$parent_status" ] && [ ! -L "$parent_status" ] || return 1 + line="$outcome [corr=$parent_corr]: inactive terminal outcome replayed: task=$task_id fingerprint=$fp" + parent_status_line_status "$parent_status" "$line" + status=$? + case "$status" in + 0) return 0 ;; + 1) return 1 ;; + *) return 2 ;; + esac +} + +reported_secondmate_receipt_valid() { + local reported=$1 parent_task_id parent_home parent_status parent_corr + prepare_pending_receipt "$reported" || return 1 + [ "$KIND" = secondmate ] || return 1 + parent_task_id=$(receipt_field "$reported" parent_task_id) || return 1 + parent_home=$(receipt_field "$reported" parent_home) || return 1 + parent_status=$(receipt_field "$reported" parent_status) || return 1 + parent_corr=$(receipt_field "$reported" parent_corr) || return 1 + [ -n "$parent_task_id" ] && [ -n "$parent_home" ] && [ -n "$parent_status" ] || return 1 + printf '%s' "$parent_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' +} + +inactive_directory_candidates() { + local directory=$1 suffix=$2 seconds=${3:-1} cursor=${4:-} cursor_path=${5:-} + local stage_path='' source_dir='' source_key='' source_path='' source_partial='' source_cookie='' + local source_complete='' source_stamp='' metadata_boundary='' legacy_stage='' path + [ -d "$directory" ] && [ ! -L "$directory" ] || return 0 + case "$seconds" in ''|*[!0-9]*|0) return 0 ;; esac + case "$cursor" in ''|0) cursor=;; *[!0-9]*) return 1 ;; esac + [ -z "$cursor_path" ] || { [ ! -L "$cursor_path" ] || return 1; } + if [ -n "$cursor_path" ]; then + source_dir="$(dirname "$cursor_path")/.inactive-outcome-enum" + if [ -e "$source_dir" ] || [ -L "$source_dir" ]; then + [ -d "$source_dir" ] && [ ! -L "$source_dir" ] || return 1 + else + mkdir "$source_dir" || return 1 + fi + source_key=$(basename "$cursor_path") + case "$source_key" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + stage_path="$source_dir/$source_key.stage" + legacy_stage="$cursor_path.stage" + [ ! -L "$stage_path" ] && [ ! -L "$legacy_stage" ] || return 1 + if [ -e "$legacy_stage" ]; then + [ -f "$legacy_stage" ] && [ ! -e "$stage_path" ] || return 1 + mv "$legacy_stage" "$stage_path" || return 1 + fi + source_path="$source_dir/$source_key.source" + source_partial="$source_path.partial" + source_cookie="$source_path.cookie" + source_complete="$source_path.complete" + source_stamp="$source_path.stamp" + for path in "$source_path" "$source_partial" "$source_cookie" \ + "$source_complete" "$source_stamp"; do + [ ! -L "$path" ] || return 1 + done + if [ "$directory" = "$STATE" ]; then + metadata_boundary=$(fm_pane_idle_meta_freshness_path "$STATE") + boundary_dir=$(dirname "$metadata_boundary") + if [ -e "$boundary_dir" ] || [ -L "$boundary_dir" ]; then + [ -d "$boundary_dir" ] && [ ! -L "$boundary_dir" ] || return 1 + else + mkdir "$boundary_dir" || return 1 + fi + [ ! -L "$metadata_boundary" ] || return 1 + fi + fi + if command -v perl >/dev/null 2>&1; then + run_bounded_child "$seconds" perl - "$directory" "$suffix" "$cursor" "$stage_path" \ + "$source_path" "$source_partial" "$source_cookie" "$source_complete" \ + "$source_stamp" "$metadata_boundary" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); +use Digest::SHA qw(sha256_hex); +use Time::HiRes qw(stat); +my ($dir, $suffix, $cursor, $stage_path) = @ARGV[0..3]; +my ($source_path, $source_partial, $source_cookie, $source_complete, $source_stamp) = @ARGV[4..8]; +my $metadata_boundary = $ARGV[9] // ''; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +sub atomic_write { + my ($path, $value) = @_; + return 0 if $path eq '' || -l $path; + my $tmp = "$path.tmp.$$"; + my $fh; + sysopen($fh, $tmp, O_WRONLY | O_CREAT | O_EXCL | $nofollow, 0600) or return 0; + binmode($fh); + if (!print($fh $value) || !close($fh)) { + unlink($tmp); + return 0; + } + if (!rename($tmp, $path)) { + unlink($tmp); + return 0; + } + return 1; +} +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path; + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} +sub directory_stamp { + if ($metadata_boundary ne '' && -f $metadata_boundary && !-l $metadata_boundary) { + open(my $bfh, '<', $metadata_boundary) or return undef; + local $/; + my $value = <$bfh> // ''; + close($bfh) or return undef; + chomp $value; + return $value if $value =~ /\A[A-Za-z0-9._-]+\z/; + } + my @info = stat($dir) or return undef; + return sha256_hex(join("\0", @info[0, 1, 2, 7, 9, 10, 11])); +} +sub read_text { + my ($path) = @_; + return '' unless $path ne '' && -e $path; + return undef if -l $path || !-f $path; + open(my $fh, '<', $path) or return undef; + local $/; + my $value = <$fh> // ''; + close($fh) or return undef; + chomp $value; + return $value; +} +my $start = directory_stamp(); +defined($start) or exit 1; +if ($source_path ne '') { + my $saved = read_text($source_stamp); + defined($saved) or exit 1 if -e $source_stamp; + if (!defined($saved) || $saved ne $start) { + unlink($source_path, $source_partial, $source_cookie, $source_complete); + atomic_write($source_stamp, "$start\n") or exit 1; + } + my $done = -e $source_complete; + if (!$done) { + my $cookie = read_text($source_cookie); + defined($cookie) or exit 1 if -e $source_cookie; + $cookie = '' unless defined($cookie) && ($cookie eq 'EOF' || $cookie =~ /^\d+\z/); + if ($cookie ne 'EOF') { + my $pfh = open_append($source_partial) or exit 1; + select((select($pfh), $| = 1)[0]); + opendir(my $dh, $dir) or exit 1; + seekdir($dh, 0 + $cookie) or exit 1 if $cookie ne ''; + while (defined(my $entry = readdir($dh))) { + my $next = telldir($dh); + defined($next) or exit 1; + if ($entry =~ /\.\Q$suffix\E\z/ && $entry !~ /[\r\n\t]/) { + my $path = "$dir/$entry"; + if (-f $path && !-l $path) { + print($pfh $path, "\n") or exit 1; + } + } + atomic_write($source_cookie, "$next\n") or exit 1; + } + closedir($dh) or exit 1; + close($pfh) or exit 1; + atomic_write($source_cookie, "EOF\n") or exit 1; + } + my $finish = directory_stamp(); + defined($finish) or exit 1; + if ($finish ne $start) { + unlink($source_path, $source_partial, $source_cookie, $source_complete); + atomic_write($source_stamp, "$finish\n") or exit 1; + exit 75; + } + if ($metadata_boundary ne '' && !-e $metadata_boundary) { + atomic_write($metadata_boundary, "$finish\n") or exit 1; + } + open(my $pfh, '<', $source_partial) or exit 1; + my %seen; + my @paths; + while (defined(my $path = <$pfh>)) { + chomp $path; + next unless $path =~ /\A\Q$dir\E\/[^\/\r\n]+\.\Q$suffix\E\z/; + next unless -f $path && !-l $path; + next if $seen{$path}++; + push @paths, $path; + } + close($pfh) or exit 1; + @paths = sort @paths; + atomic_write($source_path, join('', map { "$_\n" } @paths)) or exit 1; + atomic_write($source_complete, "complete\n") or exit 1; + unlink($source_partial, $source_cookie) or exit 1; + } + open(my $sfh, '<', $source_path) or exit 1; + my @paths; + while (defined(my $path = <$sfh>)) { + chomp $path; + push @paths, $path if $path =~ /\A\Q$dir\E\/[^\/\r\n]+\.\Q$suffix\E\z/; + } + close($sfh) or exit 1; + my $index = $cursor eq '' ? 0 : 0 + $cursor; + while ($index < @paths) { + my $path = $paths[$index]; + print "$path\0" if -f $path && !-l $path; + $index++; + atomic_write($stage_path, "$index\n") or exit 1 if $stage_path ne ''; + } + atomic_write($stage_path, "EOF\n") or exit 1 if $stage_path ne ''; +} else { + opendir(my $dh, $dir) or exit 1; + while (defined(my $entry = readdir($dh))) { + next unless $entry =~ /\.\Q$suffix\E\z/ && $entry !~ /[\r\n\t]/; + my $path = "$dir/$entry"; + print "$path\0" if -f $path && !-l $path; + } + closedir($dh) or exit 1; +} +PERL + rc=$? + [ "$rc" = 75 ] && return 124 + return "$rc" + else + return 124 + fi +} + +inactive_enum_cursor_promote() { + local path=$1 stage value progress_dir + progress_dir="$(dirname "$path")/.inactive-outcome-enum" + stage="$progress_dir/$(basename "$path").stage" + [ ! -L "$path" ] && [ ! -L "$stage" ] || return 1 + [ -e "$stage" ] || return 0 + value=$(inactive_text_cursor_read "$stage") || return 1 + inactive_text_cursor_write "$path" "$value" || return 1 + rm -f "$stage" || return 1 +} + +inactive_enum_cursor_discard() { + local path=$1 stage legacy_stage progress_dir + progress_dir="$(dirname "$path")/.inactive-outcome-enum" + stage="$progress_dir/$(basename "$path").stage" + legacy_stage="$path.stage" + [ ! -L "$stage" ] && [ ! -L "$legacy_stage" ] || return 1 + rm -f "$stage" "$legacy_stage" +} + +receipt_candidates() { + local suffix=$1 seconds=${2:-1} cursor=${3:-} cursor_path=${4:-} + inactive_directory_candidates "$OUTCOME_DIR" "$suffix" "$seconds" "$cursor" "$cursor_path" +} + +inactive_find_candidates() { + local seconds=$1 cursor=${2:-} cursor_path=${3:-} + inactive_directory_candidates "$STATE" "meta" "$seconds" "$cursor" "$cursor_path" +} + +repair_reported_secondmate_routes() { + local scan_deadline=$1 reported kind corr parent_task_id parent_home parent_status remaining status=0 + local cursor='' cursor_found=0 started=1 pass base last processed=0 cursor_tmp candidate_tmp enum_rc enum_deferred=0 + local enum_cursor='' enum_incomplete=0 enum_source_drained=0 enum_complete=0 enum_persisted=0 + local candidate_pending=0 candidate_retain=0 item_status batch_consumed=0 batch_complete=1 retry_path= + local candidate_pending_offset=0 pending_skip=0 candidate_tmp_owned=0 + MAINTENANCE_ENUM_DEFERRED=0 + MAINTENANCE_ENUM_FAILED=0 + MAINTENANCE_DEADLINE_EXPIRED=0 + MAINTENANCE_RETRYABLE=0 + MAINTENANCE_ITEMS_PROCESSED=0 + [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 + if [ -e "$REPORTED_ROUTE_ENUM_CURSOR" ] || [ -L "$REPORTED_ROUTE_ENUM_CURSOR" ]; then + [ -f "$REPORTED_ROUTE_ENUM_CURSOR" ] && [ ! -L "$REPORTED_ROUTE_ENUM_CURSOR" ] || return 1 + enum_incomplete=1 + enum_cursor=$(inactive_text_cursor_read "$REPORTED_ROUTE_ENUM_CURSOR") || return 1 + case "$enum_cursor" in + ''|0) enum_cursor=;; + *[!0-9]*) enum_cursor=;; + esac + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + return 1 + fi + if [ ! -e "$REPORTED_ROUTE_PENDING" ] && [ ! -L "$REPORTED_ROUTE_PENDING" ] \ + && inactive_pending_retry_exists "$REPORTED_ROUTE_RETRY"; then + inactive_pending_retry_restore "$REPORTED_ROUTE_RETRY" "$REPORTED_ROUTE_PENDING" || return 1 + inactive_pending_offset_write "$REPORTED_ROUTE_OFFSET" 0 || return 1 + fi + if [ -e "$REPORTED_ROUTE_PENDING" ] || [ -L "$REPORTED_ROUTE_PENDING" ]; then + [ -f "$REPORTED_ROUTE_PENDING" ] && [ ! -L "$REPORTED_ROUTE_PENDING" ] || { + return 1 + } + if [ -s "$REPORTED_ROUTE_PENDING" ]; then + inactive_pending_merge_retry "$REPORTED_ROUTE_PENDING" "$REPORTED_ROUTE_RETRY" "$scan_deadline" + enum_rc=$? + case "$enum_rc" in + 0) ;; + 124) + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + enum_deferred=1 + batch_complete=0 + candidate_retain=1 + ;; + *) return 1 ;; + esac + candidate_tmp="$REPORTED_ROUTE_PENDING" + candidate_pending_offset=$(inactive_pending_offset "$REPORTED_ROUTE_OFFSET") || { + return 1 + } + pending_skip=$candidate_pending_offset + candidate_pending=1 + candidate_retain=1 + else + inactive_pending_clear "$REPORTED_ROUTE_PENDING" "$REPORTED_ROUTE_OFFSET" || return 1 + fi + fi + if [ "$candidate_pending" = 0 ]; then + candidate_tmp=$(mktemp "$STATE/.inactive-outcome-candidates.XXXXXX") || return 1 + [ -f "$candidate_tmp" ] && [ ! -L "$candidate_tmp" ] || { + rm -f "$candidate_tmp" + return 1 + } + candidate_tmp_owned=1 + fi + cursor=$(cat "$REPORTED_ROUTE_CURSOR" 2>/dev/null || true) + case "$cursor" in + '') ;; + *.reported) case "$cursor" in *[!A-Za-z0-9._-]*) cursor=;; esac ;; + *) cursor=;; + esac + if [ -n "$cursor" ] && { [ ! -f "$OUTCOME_DIR/$cursor" ] || [ -L "$OUTCOME_DIR/$cursor" ]; }; then + cursor= + fi + if [ "$candidate_pending" = 1 ]; then + cursor= + cursor_found=1 + started=1 + else + [ -n "$cursor" ] && started=0 + fi + for pass in 1 2; do + [ "$candidate_pending" = 0 ] || [ "$pass" = 1 ] || break + if [ "$pass" = 2 ]; then + [ -n "$cursor" ] || break + started=1 + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + break + fi + batch_consumed=0 + [ "$candidate_pending" = 1 ] && batch_consumed=$candidate_pending_offset + batch_complete=1 + [ "$enum_deferred" = 0 ] || batch_complete=0 + enum_rc=0 + if [ "$candidate_pending" = 1 ]; then + : + else + receipt_candidates reported "$remaining" "$enum_cursor" "$REPORTED_ROUTE_ENUM_CURSOR" \ + | fm_nofollow_write "$candidate_tmp" || enum_rc=$? + if [ "$enum_rc" = 0 ]; then + enum_complete=1 + enum_incomplete=0 + elif [ "$enum_rc" = 124 ]; then + enum_incomplete=1 + fi + fi + if [ "$enum_rc" -ne 0 ]; then + if [ "$enum_rc" != 124 ]; then + inactive_enum_cursor_discard "$REPORTED_ROUTE_ENUM_CURSOR" || return 1 + fi + status=1 + if [ "$enum_rc" = 124 ]; then + MAINTENANCE_ENUM_DEFERRED=1 + enum_deferred=1 + batch_complete=0 + else + MAINTENANCE_ENUM_FAILED=1 + fi + fi + while [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && IFS= read -r -d '' reported; do + if [ "$pending_skip" -gt 0 ]; then + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + batch_complete=0 + break 2 + fi + pending_skip=$((pending_skip - 1)) + continue + fi + case "$reported" in + "$OUTCOME_DIR"/*/*) + batch_consumed=$((batch_consumed + 1)) + continue + ;; + esac + base=${reported##*/} + if [ "$pass" = 1 ] && [ -n "$cursor" ] && [ "$started" = 0 ]; then + batch_consumed=$((batch_consumed + 1)) + if [ "$base" = "$cursor" ]; then + started=1 + cursor_found=1 + fi + continue + fi + if [ "$pass" = 2 ] && [ "$processed" -gt 0 ] && [ -n "$cursor" ] \ + && [ "$cursor_found" = 1 ] && [ "$base" = "$cursor" ]; then + break + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + batch_complete=0 + break 2 + fi + batch_consumed=$((batch_consumed + 1)) + if [ ! -e "$reported" ] && [ ! -L "$reported" ]; then + last=$base + processed=$((processed + 1)) + if [ "$processed" -ge "$REPORTED_ROUTE_REPAIR_LIMIT" ]; then + batch_complete=0 + break 2 + fi + continue + fi + if [ ! -e "$reported" ] && [ ! -L "$reported" ]; then + last=$base + processed=$((processed + 1)) + if [ "$processed" -ge "$REPORTED_ROUTE_REPAIR_LIMIT" ]; then + batch_complete=0 + break 2 + fi + continue + fi + [ -f "$reported" ] && [ ! -L "$reported" ] || { + status=1 + candidate_retain=1 + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + MAINTENANCE_ENUM_FAILED=1 + break 2 + } + kind=$(receipt_field "$reported" kind 2>/dev/null || true) + if [ ! -e "$reported" ] && [ ! -L "$reported" ]; then + last=$base + processed=$((processed + 1)) + if [ "$processed" -ge "$REPORTED_ROUTE_REPAIR_LIMIT" ]; then + batch_complete=0 + break 2 + fi + continue + fi + item_status=0 + case "$kind" in + ship|scout) : ;; + secondmate) + if ! reported_secondmate_receipt_valid "$reported"; then + if [ ! -e "$reported" ] && [ ! -L "$reported" ]; then + item_status=0 + else + item_status=1 + fi + else + corr=$(receipt_field "$reported" parent_corr) + parent_task_id=$(receipt_field "$reported" parent_task_id) + parent_home=$(receipt_field "$reported" parent_home) + parent_status=$(receipt_field "$reported" parent_status) + if [ ! -e "$reported" ] && [ ! -L "$reported" ]; then + item_status=0 + else + FM_LOCK_WAIT_SECS="$remaining" fm_pending_reply_secondmate_route_clear_reported \ + "$FM_HOME" "$corr" "$parent_task_id" "$parent_home" "$parent_status" || item_status=$? + fi + fi + ;; + *) item_status=1 ;; + esac + if [ "$item_status" != 0 ]; then + status=1 + [ "$item_status" = 75 ] && MAINTENANCE_RETRYABLE=1 + candidate_retain=1 + if [ -z "$retry_path" ]; then + retry_path=$REPORTED_ROUTE_RETRY + if ! inactive_retry_file_prepare "$retry_path"; then + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + MAINTENANCE_ENUM_FAILED=1 + break 2 + fi + fi + inactive_append_nul_value "$retry_path" "$reported" "$maintenance_deadline" + append_rc=$? + if [ "$append_rc" -ne 0 ]; then + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + if [ "$append_rc" = 124 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + else + MAINTENANCE_ENUM_FAILED=1 + fi + break 2 + fi + continue + fi + last=$base + processed=$((processed + 1)) + if [ "$processed" -ge "$REPORTED_ROUTE_REPAIR_LIMIT" ]; then + batch_complete=0 + break 2 + fi + done < "$candidate_tmp" + [ "$enum_deferred" = 0 ] || break + done + if [ "$candidate_pending" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && [ "$enum_deferred" = 0 ] \ + && [ "$enum_incomplete" = 1 ]; then + inactive_pending_clear "$REPORTED_ROUTE_PENDING" "$REPORTED_ROUTE_OFFSET" || status=1 + MAINTENANCE_ENUM_DEFERRED=1 + enum_deferred=1 + enum_source_drained=1 + status=1 + fi + if [ "$enum_source_drained" = 1 ]; then + : + elif [ "$candidate_pending" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && [ "$enum_deferred" = 0 ]; then + inactive_pending_clear "$REPORTED_ROUTE_PENDING" "$REPORTED_ROUTE_OFFSET" || status=1 + elif [ "$enum_deferred" = 1 ] || [ "$candidate_retain" = 1 ] || [ "$candidate_pending" = 1 ]; then + if inactive_persist_nul_suffix "$candidate_tmp" "$REPORTED_ROUTE_PENDING" "$batch_consumed" "" "$scan_deadline"; then + enum_persisted=1 + else + enum_rc=$? + if [ "$enum_rc" = 124 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + else + status=1 + fi + fi + fi + if [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && { + [ "$enum_persisted" = 1 ] || { + [ "$enum_complete" = 1 ] && [ "$candidate_retain" = 0 ] + } + }; then + inactive_enum_cursor_promote "$REPORTED_ROUTE_ENUM_CURSOR" || status=1 + [ "$status" = 0 ] && rm -f "$REPORTED_ROUTE_ENUM_CURSOR" || true + else + inactive_enum_cursor_discard "$REPORTED_ROUTE_ENUM_CURSOR" || status=1 + fi + if [ "$candidate_tmp_owned" = 1 ]; then + rm -f "$candidate_tmp" || status=1 + fi + if [ "$processed" -gt 0 ]; then + if cursor_tmp=$(mktemp "$STATE/.reported-route-repair.cursor.XXXXXX"); then + if [ ! -f "$cursor_tmp" ] || [ -L "$cursor_tmp" ] \ + || ! printf '%s\n' "$last" | fm_nofollow_write "$cursor_tmp" \ + || ! fm_nofollow_rename "$cursor_tmp" "$REPORTED_ROUTE_CURSOR"; then + status=1 + rm -f "$cursor_tmp" + fi + else + status=1 + fi + fi + MAINTENANCE_ITEMS_PROCESSED=$processed + return "$status" +} + +republish_pending_receipt() { + local pending=$1 task_lock meta window backend current_incarnation current_kind status=0 rc + local deadline_ms=${FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS:-} now_ms remaining_ms wait_secs + FM_WAKE_APPEND_CREATED=0 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + if ! prepare_pending_receipt "$pending"; then + [ ! -e "$pending" ] && [ ! -L "$pending" ] && return 0 + return 1 + fi + case "$KIND" in ship|scout|secondmate) ;; *) return 1 ;; esac + meta="$STATE/$ID.meta" + [ -f "$meta" ] && [ ! -L "$meta" ] || return 75 + herdr_identity_allowed "$meta" || return 75 + task_lock="$STATE/.spawn-$ID.lock" + FM_LOCK_WAIT_SECS="${FM_LOCK_WAIT_SECS:-30}" fm_lock_acquire_wait "$task_lock" || return 75 + if [ ! -f "$meta" ] || [ -L "$meta" ] \ + || ! herdr_identity_allowed "$meta"; then + status=75 + else + if current_kind=$(meta_value_unique "$meta" kind 2>/dev/null); then + : + else + rc=$? + [ "$rc" = 1 ] || status=75 + current_kind=ship + fi + if [ "$status" = 0 ]; then + case "$KIND:$current_kind" in + ship:ship|scout:scout|secondmate:ship|secondmate:scout) : ;; + *) status=75 ;; + esac + fi + fi + if [ "$status" = 0 ] && current_incarnation=$(read_incarnation "$meta" "$ID" 2>/dev/null); then + [ "$current_incarnation" = "$INC" ] || status=75 + if [ "$status" = 0 ]; then + window=$(meta_value_unique "$meta" window 2>/dev/null) || status=75 + if backend=$(meta_value_unique "$meta" backend 2>/dev/null); then + case "$backend" in tmux|herdr) ;; *) status=75 ;; esac + else + rc=$? + [ "$rc" = 1 ] && backend=tmux || status=75 + fi + fi + if [ "$status" = 0 ]; then + if [ -n "$deadline_ms" ]; then + now_ms=$(clock_millis) + remaining_ms=$((deadline_ms - now_ms)) + [ "$remaining_ms" -gt 0 ] || status=75 + wait_secs=$((remaining_ms / 1000)) + else + wait_secs=${FM_LOCK_WAIT_SECS:-30} + fi + if [ "$status" = 0 ]; then + if [ "$KIND" = secondmate ]; then + FM_LOCK_WAIT_SECS="$wait_secs" publish_secondmate_receipt_and_wake \ + "$meta" "$ID" "$window" "$backend" "$current_incarnation" "$SOURCE" || status=$? + else + FM_LOCK_WAIT_SECS="$wait_secs" republish_existing_receipt_wake \ + "$meta" "$ID" "$window" "$backend" "$current_incarnation" "$SOURCE" || status=$? + fi + fi + fi + else + status=75 + fi + fm_lock_release "$task_lock" || status=1 + return "$status" +} + +republish_pending_receipts() { + local scan_deadline=$1 pending remaining status=0 + local cursor='' cursor_found=0 pass started=1 + local base last='' processed=0 cursor_tmp candidate_tmp enum_rc enum_deferred=0 + local enum_cursor='' enum_incomplete=0 enum_source_drained=0 enum_complete=0 enum_persisted=0 + local candidate_pending=0 candidate_retain=0 batch_consumed=0 batch_complete=1 retry_path= + local candidate_pending_offset=0 pending_skip=0 candidate_tmp_owned=0 + local LC_ALL=C + MAINTENANCE_ENUM_DEFERRED=0 + MAINTENANCE_ENUM_FAILED=0 + MAINTENANCE_RETRYABLE=0 + MAINTENANCE_DEADLINE_EXPIRED=0 + MAINTENANCE_ITEMS_PROCESSED=0 + [ -d "$OUTCOME_DIR" ] && [ ! -L "$OUTCOME_DIR" ] || return 0 + if [ -e "$PENDING_RECEIPT_ENUM_CURSOR" ] || [ -L "$PENDING_RECEIPT_ENUM_CURSOR" ]; then + [ -f "$PENDING_RECEIPT_ENUM_CURSOR" ] && [ ! -L "$PENDING_RECEIPT_ENUM_CURSOR" ] || return 1 + enum_incomplete=1 + enum_cursor=$(inactive_text_cursor_read "$PENDING_RECEIPT_ENUM_CURSOR") || return 1 + case "$enum_cursor" in + ''|0) enum_cursor=;; + *[!0-9]*) enum_cursor=;; + esac + fi + if [ ! -e "$PENDING_RECEIPT_PENDING" ] && [ ! -L "$PENDING_RECEIPT_PENDING" ] \ + && inactive_pending_retry_exists "$PENDING_RECEIPT_RETRY"; then + inactive_pending_retry_restore "$PENDING_RECEIPT_RETRY" "$PENDING_RECEIPT_PENDING" || return 1 + inactive_pending_offset_write "$PENDING_RECEIPT_OFFSET" 0 || return 1 + fi + if [ -e "$PENDING_RECEIPT_PENDING" ] || [ -L "$PENDING_RECEIPT_PENDING" ]; then + [ -f "$PENDING_RECEIPT_PENDING" ] && [ ! -L "$PENDING_RECEIPT_PENDING" ] || { + return 1 + } + if [ -s "$PENDING_RECEIPT_PENDING" ]; then + inactive_pending_merge_retry "$PENDING_RECEIPT_PENDING" "$PENDING_RECEIPT_RETRY" "$scan_deadline" + enum_rc=$? + case "$enum_rc" in + 0) ;; + 124) + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + enum_deferred=1 + batch_complete=0 + candidate_retain=1 + ;; + *) return 1 ;; + esac + candidate_tmp="$PENDING_RECEIPT_PENDING" + candidate_pending_offset=$(inactive_pending_offset "$PENDING_RECEIPT_OFFSET") || { + return 1 + } + pending_skip=$candidate_pending_offset + candidate_pending=1 + candidate_retain=1 + else + inactive_pending_clear "$PENDING_RECEIPT_PENDING" "$PENDING_RECEIPT_OFFSET" || return 1 + fi + fi + if [ "$candidate_pending" = 0 ]; then + candidate_tmp=$(mktemp "$STATE/.inactive-outcome-candidates.XXXXXX") || return 1 + [ -f "$candidate_tmp" ] && [ ! -L "$candidate_tmp" ] || { + rm -f "$candidate_tmp" + return 1 + } + candidate_tmp_owned=1 + fi + cursor=$(cat "$PENDING_RECEIPT_CURSOR" 2>/dev/null || true) + case "$cursor" in + '') ;; + *.pending) + base=${cursor%.pending} + case "$base" in ''|*[!A-Fa-f0-9]*) cursor=;; esac + ;; + *) cursor=;; + esac + if [ "$candidate_pending" = 1 ]; then + cursor= + cursor_found=1 + started=1 + else + [ -n "$cursor" ] && started=0 + fi + for pass in 1 2; do + [ "$candidate_pending" = 0 ] || [ "$pass" = 1 ] || break + if [ "$pass" = 2 ]; then + [ -n "$cursor" ] || break + started=1 + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + break + fi + batch_consumed=0 + [ "$candidate_pending" = 1 ] && batch_consumed=$candidate_pending_offset + batch_complete=1 + [ "$enum_deferred" = 0 ] || batch_complete=0 + enum_rc=0 + if [ "$candidate_pending" = 1 ]; then + : + else + receipt_candidates pending "$remaining" "$enum_cursor" "$PENDING_RECEIPT_ENUM_CURSOR" \ + | fm_nofollow_write "$candidate_tmp" || enum_rc=$? + if [ "$enum_rc" = 0 ]; then + enum_complete=1 + enum_incomplete=0 + elif [ "$enum_rc" = 124 ]; then + enum_incomplete=1 + fi + fi + if [ "$enum_rc" -ne 0 ]; then + if [ "$enum_rc" != 124 ]; then + inactive_enum_cursor_discard "$PENDING_RECEIPT_ENUM_CURSOR" || return 1 + fi + status=1 + if [ "$enum_rc" = 124 ]; then + MAINTENANCE_ENUM_DEFERRED=1 + enum_deferred=1 + batch_complete=0 + else + MAINTENANCE_ENUM_FAILED=1 + fi + fi + while [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && IFS= read -r -d '' pending; do + if [ "$pending_skip" -gt 0 ]; then + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + batch_complete=0 + break 2 + fi + pending_skip=$((pending_skip - 1)) + continue + fi + case "$pending" in + "$OUTCOME_DIR"/*/*) + batch_consumed=$((batch_consumed + 1)) + continue + ;; + esac + base=${pending##*/} + if [ "$pass" = 1 ] && [ -n "$cursor" ] && [ "$started" = 0 ]; then + batch_consumed=$((batch_consumed + 1)) + if [ "$base" = "$cursor" ]; then + started=1 + cursor_found=1 + continue + fi + continue + fi + if [ "$pass" = 2 ] && [ "$processed" -gt 0 ] && [ -n "$cursor" ] \ + && [ "$cursor_found" = 1 ] && [ "$base" = "$cursor" ]; then + break + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + status=1 + batch_complete=0 + break 2 + fi + batch_consumed=$((batch_consumed + 1)) + if FM_LOCK_WAIT_SECS="$remaining" FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS="$scan_deadline" \ + republish_pending_receipt "$pending"; then + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$ID" "$OUTCOME" "$FP" + fi + else + enum_rc=$? + status=1 + if [ "$enum_rc" = 75 ]; then + MAINTENANCE_RETRYABLE=1 + candidate_retain=1 + if [ -z "$retry_path" ]; then + retry_path=$PENDING_RECEIPT_RETRY + if ! inactive_retry_file_prepare "$retry_path"; then + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + MAINTENANCE_ENUM_FAILED=1 + break 2 + fi + fi + inactive_append_nul_value "$retry_path" "$pending" "$maintenance_deadline" + append_rc=$? + if [ "$append_rc" -ne 0 ]; then + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + if [ "$append_rc" = 124 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + else + MAINTENANCE_ENUM_FAILED=1 + fi + break 2 + fi + continue + fi + candidate_retain=1 + batch_consumed=$((batch_consumed - 1)) + batch_complete=0 + MAINTENANCE_ENUM_FAILED=1 + break 2 + fi + last=$base + processed=$((processed + 1)) + if [ "$processed" -ge "$PENDING_RECEIPT_REPUBLISH_LIMIT" ]; then + batch_complete=0 + break 2 + fi + done < "$candidate_tmp" + [ "$enum_deferred" = 0 ] || break + done + if [ "$candidate_pending" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && [ "$enum_deferred" = 0 ] \ + && [ "$enum_incomplete" = 1 ]; then + inactive_pending_clear "$PENDING_RECEIPT_PENDING" "$PENDING_RECEIPT_OFFSET" || status=1 + MAINTENANCE_ENUM_DEFERRED=1 + enum_deferred=1 + enum_source_drained=1 + status=1 + fi + if [ "$enum_source_drained" = 1 ]; then + : + elif [ "$candidate_pending" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && [ "$enum_deferred" = 0 ]; then + inactive_pending_clear "$PENDING_RECEIPT_PENDING" "$PENDING_RECEIPT_OFFSET" || status=1 + elif [ "$enum_deferred" = 1 ] || [ "$candidate_retain" = 1 ] || [ "$candidate_pending" = 1 ]; then + if inactive_persist_nul_suffix "$candidate_tmp" "$PENDING_RECEIPT_PENDING" "$batch_consumed" "" "$scan_deadline"; then + enum_persisted=1 + else + enum_rc=$? + if [ "$enum_rc" = 124 ]; then + MAINTENANCE_DEADLINE_EXPIRED=1 + else + status=1 + fi + fi + fi + if [ "$MAINTENANCE_ENUM_FAILED" = 0 ] && { + [ "$enum_persisted" = 1 ] || { + [ "$enum_complete" = 1 ] && [ "$candidate_retain" = 0 ] + } + }; then + inactive_enum_cursor_promote "$PENDING_RECEIPT_ENUM_CURSOR" || status=1 + [ "$status" = 0 ] && rm -f "$PENDING_RECEIPT_ENUM_CURSOR" || true + else + inactive_enum_cursor_discard "$PENDING_RECEIPT_ENUM_CURSOR" || status=1 + fi + if [ "$candidate_tmp_owned" = 1 ]; then + rm -f "$candidate_tmp" || status=1 + fi + if [ "$processed" -gt 0 ]; then + if cursor_tmp=$(mktemp "$STATE/.pending-receipt-republish.cursor.XXXXXX"); then + if [ ! -f "$cursor_tmp" ] || [ -L "$cursor_tmp" ] \ + || ! printf '%s\n' "$last" | fm_nofollow_write "$cursor_tmp" \ + || ! fm_nofollow_rename "$cursor_tmp" "$PENDING_RECEIPT_CURSOR"; then + status=1 + rm -f "$cursor_tmp" + fi + else + status=1 + fi + fi + MAINTENANCE_ITEMS_PROCESSED=$processed + return "$status" +} + +read_incarnation() { fm_pane_idle_read_incarnation "$@"; } + +run_step_incarnation_evidence_path() { + if [ "$#" = 1 ]; then + printf '%s/.run-step-incarnation-%s' "$STATE" "$1" + else + printf '%s/.run-step-incarnation-%s.%s' "$STATE" "$1" "$2" + fi +} + +run_step_incarnation_evidence_valid() { + local id=$1 incarnation=$2 outcome=$3 snapshot=$4 evidence run_id + valid_task_id "$id" || return 1 + case "$outcome" in done|failed) ;; *) return 1 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + [ -n "$snapshot" ] || return 1 + evidence=$(run_step_incarnation_evidence_path "$id") || return 1 + if [ ! -f "$evidence" ] || [ -L "$evidence" ] \ + || [ "$(meta_value_unique "$evidence" spawn_incarnation 2>/dev/null || true)" != "$incarnation" ]; then + evidence=$(run_step_incarnation_evidence_path "$id" "$incarnation") || return 1 + fi + [ -f "$evidence" ] && [ ! -L "$evidence" ] || return 1 + fm_nofollow_read "$evidence" | awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["task_id"]=1; allowed["run_id"]=1 + allowed["spawn_incarnation"]=1; allowed["state"]=1 + valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + values[key]=substr($0, index($0, "=") + 1) + next + } + { valid=0 } + END { + if (!("schema" in seen) || !("task_id" in seen) || !("run_id" in seen) \ + || !("spawn_incarnation" in seen) || !("state" in seen)) valid=0 + exit !(valid && values["schema"] == "fm-jt-run-step-incarnation.v1" \ + && values["state"] == "active") + } + ' 2>/dev/null || return 1 + [ "$(meta_value_unique "$evidence" schema 2>/dev/null)" = fm-jt-run-step-incarnation.v1 ] || return 1 + [ "$(meta_value_unique "$evidence" task_id 2>/dev/null)" = "$id" ] || return 1 + run_id=$(meta_value_unique "$evidence" run_id 2>/dev/null) || return 1 + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + [ -n "$(printf '%s' "$snapshot" | sed -n 's/.* · run-id=\([^ ·]*\)$/\1/p')" ] || return 1 + [ "$run_id" = "$(printf '%s' "$snapshot" | sed -n 's/.* · run-id=\([^ ·]*\)$/\1/p')" ] || return 1 + [ "$(meta_value_unique "$evidence" spawn_incarnation 2>/dev/null)" = "$incarnation" ] || return 1 +} + +terminal_snapshot_matches_current() { + local id=$1 expected=$2 status_snapshot=$3 state_timeout state_tmp state_rc current + local lock_path=${CHILD_LOCK:-${FM_TASK_LOCK_PATH:-}} lock_owner=${CHILD_LOCK_OWNER:-${FM_TASK_LOCK_OWNER:-}} + case "$expected" in + done:*|failed:*) [ "$expected" = "$status_snapshot" ]; return $? ;; + state:\ done\ *|state:\ failed\ *) ;; + *) return 1 ;; + esac + state_timeout=$(bounded_secs "${FM_INACTIVE_OUTCOME_STATE_TIMEOUT_SECS:-10}" 10 1 300) + state_tmp=$(mktemp "$STATE/.inactive-terminal-snapshot.XXXXXX") || return 1 + state_rc=0 + if [ "${CHILD_LOCK_HELD:-0}" = 1 ]; then + [ -n "$lock_path" ] && [ -n "$lock_owner" ] || { + rm -f "$state_tmp" + return 1 + } + run_bounded_child "$state_timeout" env \ + FM_CREW_STATE_NM_TIMEOUT="$state_timeout" \ + FM_TASK_LOCK_PATH="$lock_path" FM_TASK_LOCK_OWNER="$lock_owner" \ + "$FM_CREW_STATE_BIN" "$id" 2>/dev/null | fm_nofollow_write "$state_tmp" || state_rc=$? + else + run_bounded_child "$state_timeout" env FM_CREW_STATE_NM_TIMEOUT="$state_timeout" \ + "$FM_CREW_STATE_BIN" "$id" 2>/dev/null | fm_nofollow_write "$state_tmp" || state_rc=$? + fi + current= + if [ "$state_rc" -eq 0 ]; then + current=$(cat "$state_tmp" 2>/dev/null) || state_rc=$? + fi + rm -f "$state_tmp" || [ "$state_rc" -ne 0 ] || state_rc=1 + [ "$state_rc" -eq 0 ] || return 1 + case "$current" in *$'\n'*) return 1 ;; esac + [ "$current" = "$expected" ] +} + +surface_retry_valid() { + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["task"]=1; allowed["snapshot"]=1 + allowed["spawn_incarnation"]=1; allowed["tasktmp"]=1 + allowed["window"]=1; allowed["worktree"]=1; allowed["wake_key"]=1 + allowed["parent_corr"]=1 + allowed["wake_published"]=1 + required["schema"]=1; required["task"]=1; required["snapshot"]=1 + required["spawn_incarnation"]=1; required["tasktmp"]=1 + required["window"]=1; required["worktree"]=1; required["wake_key"]=1 + required["parent_corr"]=1 + required["wake_published"]=1 + valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + values[key]=substr($0, index($0, "=") + 1) + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !(valid && values["schema"] == "fm-hb-surface-retry.v1" && values["task"] != "" && values["snapshot"] != "" && values["wake_key"] != "" && values["wake_published"] ~ /^[012]$/) + } + ' "$1" 2>/dev/null +} + +surface_retry_matches_current() { + local retry=$1 id=$2 meta=$3 status_file current_snapshot saved_snapshot saved_spawn + local current_spawn current_tasktmp current_window current_worktree current_parent_corr saved_parent_corr rc + [ -f "$retry" ] && [ ! -L "$retry" ] || return 1 + surface_retry_valid "$retry" || return 1 + [ "$(meta_value_unique "$retry" task 2>/dev/null)" = "$id" ] || return 1 + status_file="$STATE/$id.status" + [ -f "$status_file" ] && [ ! -L "$status_file" ] || return 1 + current_snapshot=$(awk 'NF { line=$0 } END { if (line == "") exit 1; print line }' "$status_file" 2>/dev/null) || return 1 + saved_snapshot=$(meta_value_unique "$retry" snapshot 2>/dev/null) || return 1 + terminal_snapshot_matches_current "$id" "$saved_snapshot" "$current_snapshot" || return 1 + saved_parent_corr=$(meta_value_unique "$retry" parent_corr 2>/dev/null || true) + current_parent_corr=$(replay_parent_corr) || return 1 + [ "$saved_parent_corr" = "$current_parent_corr" ] || return 1 + saved_spawn=$(meta_value_unique "$retry" spawn_incarnation 2>/dev/null) || return 1 + if current_spawn=$(meta_value_unique "$meta" spawn_incarnation 2>/dev/null); then + [ "$saved_spawn" = "$current_spawn" ] || return 1 + return 0 + fi + rc=$? + [ "$rc" = 1 ] || return 1 + [ -z "$saved_spawn" ] || return 1 + current_tasktmp=$(meta_value "$meta" tasktmp) + current_window=$(meta_value "$meta" window) + current_worktree=$(meta_value "$meta" worktree) + [ "$(meta_value_unique "$retry" tasktmp 2>/dev/null)" = "$current_tasktmp" ] || return 1 + [ "$(meta_value_unique "$retry" window 2>/dev/null)" = "$current_window" ] || return 1 + [ "$(meta_value_unique "$retry" worktree 2>/dev/null)" = "$current_worktree" ] || return 1 +} + +surface_retry_mark_published() { + local retry=$1 tmp line seen=0 + tmp=$(mktemp "$STATE/.hb-surface-retry.XXXXXX") || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + wake_published=*) printf 'wake_published=1\n'; seen=1 ;; + *) printf '%s\n' "$line" ;; + esac + done < "$retry" + [ "$seen" = 1 ] || printf 'wake_published=1\n' + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + [ ! -L "$retry" ] || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$retry" || { rm -f "$tmp"; return 1; } +} + +surface_retry_wake_present() { + local wake_key=$1 + [ -f "$FM_WAKE_QUEUE" ] && [ ! -L "$FM_WAKE_QUEUE" ] || return 1 + command -v perl >/dev/null 2>&1 || return 1 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path, $wanted) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 2; + sysopen(my $fh, $path, O_RDONLY | $nofollow) or exit 2; + while (defined(my $line = <$fh>)) { + chomp $line; + my @fields = split(/\t/, $line, -1); + next unless @fields == 5; + next unless $fields[0] =~ /\A[0-9]+\z/ && $fields[1] =~ /\A[0-9]+\z/; + next unless $fields[2] =~ /\A(?:signal|stale|check|heartbeat)\z/; + next unless defined($fields[4]) && $fields[4] ne ''; + exit 0 if $fields[3] eq $wanted; + } + close($fh) or exit 2; + exit 1; + ' "$FM_WAKE_QUEUE" "$wake_key" +} + +surface_retry_remove() { + local retry=$1 + [ ! -L "$retry" ] || return 1 + rm -f "$retry" +} + +surface_retry_published_current() { + local id=$1 meta=$2 retry=$3 wake_key=$4 published fp suffix receipt_state= + if [ -L "$retry" ] || [ -e "$retry" ]; then + [ -f "$retry" ] && [ ! -L "$retry" ] && surface_retry_valid "$retry" || return 2 + else + return 1 + fi + surface_retry_matches_current "$retry" "$id" "$meta" || return 1 + [ "$(meta_value_unique "$retry" wake_key 2>/dev/null)" = "$wake_key" ] || return 2 + published=$(meta_value_unique "$retry" wake_published 2>/dev/null) || return 2 + fp=${wake_key#inactive-outcome:} + case "$fp" in ''|*[!A-Fa-f0-9]*) return 2 ;; esac + for suffix in pending presented reported; do + receipt_state= + [ ! -L "$(receipt_path "$fp" "$suffix")" ] || return 2 + if [ -f "$(receipt_path "$fp" "$suffix")" ]; then + receipt_state=$suffix + break + fi + done + case "$published" in + 1) + [ -n "$receipt_state" ] || return 1 + case "$receipt_state" in + presented|reported) + if ! surface_retry_wake_present "$wake_key"; then + surface_retry_remove "$retry" || return 2 + fi + ;; + pending) surface_retry_wake_present "$wake_key" || return 1 ;; + *) return 2 ;; + esac + return 0 + ;; + 2) + [ -n "$receipt_state" ] || return 1 + case "$receipt_state" in + presented|reported) + if surface_retry_wake_present "$wake_key"; then + surface_retry_mark_published "$retry" || return 2 + else + surface_retry_remove "$retry" || return 2 + fi + return 0 + ;; + pending) + surface_retry_wake_present "$wake_key" || return 1 + surface_retry_mark_published "$retry" || return 2 + return 0 + ;; + *) return 2 ;; + esac + ;; + 0) ;; + *) return 2 ;; + esac + surface_retry_wake_present "$wake_key" || return 1 + surface_retry_mark_published "$retry" || return 2 +} + +terminal_outcome_surfaced() { + local id=$1 meta=$2 outcome=$3 wake_key=${4:-} + local key raw raw_outcome marker retry marker_snapshot marker_spawn marker_parent_corr current_parent_corr current_snapshot retry_status + local current_spawn current_tasktmp current_window current_worktree status_file + key=$(printf '%s' "$id" | tr ':/.' '___') + current_parent_corr=$(replay_parent_corr) || return 2 + raw="$STATE/.hb-surfaced-$key" + marker="$STATE/.hb-terminal-surfaced-$key" + retry="$STATE/.hb-surface-retry-$key" + if [ -n "$wake_key" ]; then + surface_retry_published_current "$id" "$meta" "$retry" "$wake_key" || retry_status=$? + case "${retry_status:-0}" in + 0) return 0 ;; + 1) ;; + *) return 2 ;; + esac + fi + [ -f "$raw" ] && [ ! -L "$raw" ] || return 1 + raw=$(cat "$raw" 2>/dev/null || true) + [ -n "$raw" ] || return 1 + case "$raw" in + done:*|failed:*) raw_outcome=${raw%%:*} ;; + state:\ 'done'\ *) raw_outcome='done' ;; + state:\ failed\ *) raw_outcome=failed ;; + *) return 1 ;; + esac + [ "$raw_outcome" = "$outcome" ] || return 1 + status_file="$STATE/$id.status" + [ -f "$status_file" ] && [ ! -L "$status_file" ] || return 1 + current_snapshot=$(awk 'NF { line=$0 } END { if (line == "") exit 1; print line }' "$status_file" 2>/dev/null) || return 1 + terminal_snapshot_matches_current "$id" "$raw" "$current_snapshot" || return 1 + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["snapshot"]=1; allowed["spawn_incarnation"]=1 + allowed["tasktmp"]=1; allowed["window"]=1; allowed["worktree"]=1 + allowed["parent_corr"]=1 + required["schema"]=1; required["snapshot"]=1; required["spawn_incarnation"]=1 + required["tasktmp"]=1; required["window"]=1; required["worktree"]=1 + required["parent_corr"]=1 + valid=1 + } + /^[^=]+=/{ + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + values[key]=substr($0, index($0, "=") + 1) + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !(valid && values["schema"] == "fm-hb-terminal-surfaced.v1") + } + ' "$marker" 2>/dev/null || return 1 + marker_parent_corr=$(meta_value_unique "$marker" parent_corr 2>/dev/null || true) + [ "$marker_parent_corr" = "$current_parent_corr" ] || return 1 + marker_snapshot=$(meta_value_unique "$marker" snapshot 2>/dev/null) || return 1 + [ "$marker_snapshot" = "$raw" ] || return 1 + marker_spawn=$(meta_value_unique "$marker" spawn_incarnation 2>/dev/null) || return 1 + current_spawn= + if current_spawn=$(meta_value_unique "$meta" spawn_incarnation 2>/dev/null); then + [ "$marker_spawn" = "$current_spawn" ] || return 1 + else + [ -z "$marker_spawn" ] || return 1 + current_tasktmp=$(meta_value "$meta" tasktmp) + current_window=$(meta_value "$meta" window) + current_worktree=$(meta_value "$meta" worktree) + [ "$(meta_value_unique "$marker" tasktmp 2>/dev/null)" = "$current_tasktmp" ] || return 1 + [ "$(meta_value_unique "$marker" window 2>/dev/null)" = "$current_window" ] || return 1 + [ "$(meta_value_unique "$marker" worktree 2>/dev/null)" = "$current_worktree" ] || return 1 + fi + return 0 +} + +replay_surface_retry_write() { + local id=$1 meta=$2 snapshot=$3 incarnation=$4 key=$5 published=$6 + local retry tmp tasktmp window worktree marker_incarnation parent_corr rc + case "$published" in 0|1|2) ;; *) return 1 ;; esac + parent_corr=$(replay_parent_corr) || return 1 + tasktmp=$(meta_value "$meta" tasktmp) + window=$(meta_value "$meta" window) + worktree=$(meta_value "$meta" worktree) + if meta_value_unique "$meta" spawn_incarnation >/dev/null; then + marker_incarnation=$incarnation + else + rc=$? + [ "$rc" = 1 ] || return 1 + marker_incarnation= + fi + retry="$STATE/.hb-surface-retry-$(printf '%s' "$id" | tr ':/.' '___')" + if [ -e "$retry" ]; then + [ -f "$retry" ] && [ ! -L "$retry" ] && surface_retry_valid "$retry" || return 1 + if surface_retry_matches_current "$retry" "$id" "$meta"; then + [ "$(meta_value_unique "$retry" wake_key 2>/dev/null)" = "$key" ] || return 2 + fi + fi + tmp=$(mktemp "$STATE/.hb-surface-retry.XXXXXX") || return 1 + if ! printf 'schema=fm-hb-surface-retry.v1\ntask=%s\nsnapshot=%s\nspawn_incarnation=%s\ntasktmp=%s\nwindow=%s\nworktree=%s\nparent_corr=%s\nwake_key=%s\nwake_published=%s\n' \ + "$id" "$snapshot" "$marker_incarnation" "$tasktmp" "$window" "$worktree" "$parent_corr" "$key" "$published" | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$retry"; then + rm -f "$tmp" + return 1 + fi +} + +replay_surface_marker() { + local id=$1 meta=$2 snapshot=$3 incarnation=$4 wake_key=$5 key raw marker retry tmp tasktmp window worktree + local marker_incarnation parent_corr rc + key=$(printf '%s' "$id" | tr ':/.' '___') + parent_corr=$(replay_parent_corr) || return 1 + raw="$STATE/.hb-surfaced-$key" + marker="$STATE/.hb-terminal-surfaced-$key" + retry="$STATE/.hb-surface-retry-$key" + tasktmp=$(meta_value "$meta" tasktmp) + window=$(meta_value "$meta" window) + worktree=$(meta_value "$meta" worktree) + if meta_value_unique "$meta" spawn_incarnation >/dev/null; then + marker_incarnation=$incarnation + else + rc=$? + [ "$rc" = 1 ] || return 1 + marker_incarnation= + fi + replay_surface_retry_write "$id" "$meta" "$snapshot" "$incarnation" "$wake_key" 1 || return 1 + tmp=$(mktemp "$STATE/.hb-terminal-surfaced.XXXXXX") || return 1 + if ! printf 'schema=fm-hb-terminal-surfaced.v1\nsnapshot=%s\nspawn_incarnation=%s\ntasktmp=%s\nwindow=%s\nworktree=%s\nparent_corr=%s\n' \ + "$snapshot" "$marker_incarnation" "$tasktmp" "$window" "$worktree" "$parent_corr" | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$marker"; then + rm -f "$tmp" + return 1 + fi + tmp=$(mktemp "$STATE/.hb-surfaced.XXXXXX") || return 1 + if ! printf '%s' "$snapshot" | fm_nofollow_write "$tmp" || ! fm_nofollow_rename "$tmp" "$raw"; then + rm -f "$tmp" + return 1 + fi + rm -f "$retry" || return 1 +} + +replay_receipt_exists() { + local suffix path + for suffix in pending presented reported; do + path=$(receipt_path "$FP" "$suffix") + [ -f "$path" ] && [ ! -L "$path" ] && return 0 + done + return 1 +} + +child_cleanup() { + local status=$? + if [ "$WAKE_QUEUE_LOCK_HELD" = 1 ]; then + wake_queue_lock_release || true + fi + if [ "$CHILD_LOCK_HELD" = 1 ]; then + fm_lock_release "$CHILD_LOCK" || true + fi + exit "$status" +} + +reconcile_child() { + local id=$1 meta="$STATE/$1.meta" kind kind_status backend window now activity age line outcome source + local snapshot token key route_rc parent_corr state_tmp state_rc existing_rc surface_status publication_status state_timeout scan_remaining child_lock_owner + valid_task_id "$id" || return 0 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + if kind=$(meta_value_unique "$meta" kind 2>/dev/null); then + : + else + kind_status=$? + [ "$kind_status" = 1 ] || return 0 + kind=ship + fi + case "$kind" in ship|scout) ;; *) return 0 ;; esac + herdr_identity_allowed "$meta" || return 0 + CHILD_LOCK="$STATE/.spawn-$id.lock" + FM_LOCK_WAIT_SECS=$(bounded_secs "${FM_INACTIVE_OUTCOME_LOCK_WAIT_SECS:-30}" 30 0 300) + fm_lock_acquire_wait "$CHILD_LOCK" || return 75 + CHILD_LOCK_HELD=1 + child_lock_owner=$(fm_lock_link_owner "$CHILD_LOCK") || return 75 + CHILD_LOCK_OWNER=$child_lock_owner + trap child_cleanup EXIT INT TERM + # Teardown/relaunch can replace or remove metadata only after the same lock is + # released. Re-read it after acquiring the lock so the snapshot belongs to the + # current incarnation. + [ -f "$meta" ] && [ ! -L "$meta" ] || return 0 + if kind=$(meta_value_unique "$meta" kind 2>/dev/null); then + : + else + kind_status=$? + [ "$kind_status" = 1 ] || return 0 + kind=ship + fi + case "$kind" in ship|scout) ;; *) return 0 ;; esac + herdr_identity_allowed "$meta" || return 0 + now=$(date +%s) + activity=$(latest_activity "$id") + [ "$activity" -gt 0 ] || return 0 + age=$((now - activity)) + [ "$age" -ge "$RECONCILE_SECS" ] || return 0 + state_tmp=$(mktemp "$STATE/.$id.inactive-state.XXXXXX") || return 1 + [ -f "$state_tmp" ] && [ ! -L "$state_tmp" ] || { rm -f "$state_tmp"; return 1; } + state_timeout=$(bounded_secs "${FM_INACTIVE_OUTCOME_STATE_TIMEOUT_SECS:-10}" 10 1 300) + scan_remaining=${FM_INACTIVE_OUTCOME_SCAN_REMAINING_SECS:-} + case "$scan_remaining" in + ''|*[!0-9]*) scan_remaining=;; + *) + while [ "${scan_remaining#0}" != "$scan_remaining" ]; do scan_remaining=${scan_remaining#0}; done + [ -n "$scan_remaining" ] || scan_remaining=0 + [ "$scan_remaining" -gt 0 ] && [ "$scan_remaining" -lt "$state_timeout" ] \ + && state_timeout=$scan_remaining + ;; + esac + state_rc=0 + export FM_CREW_STATE_NM_TIMEOUT="$state_timeout" + run_bounded_child "$state_timeout" env \ + FM_TASK_LOCK_PATH="$CHILD_LOCK" FM_TASK_LOCK_OWNER="$child_lock_owner" \ + "$FM_CREW_STATE_BIN" "$id" \ + 2>/dev/null | fm_nofollow_write "$state_tmp" || state_rc=$? + line= + if [ "$state_rc" -eq 0 ]; then + line=$(cat "$state_tmp" 2>/dev/null) || state_rc=$? + fi + rm -f "$state_tmp" || [ "$state_rc" -ne 0 ] || state_rc=1 + [ "$state_rc" -eq 0 ] || return "$state_rc" + case "$line" in *$'\n'*) return 0 ;; esac + case "$line" in + state:\ done\ *|state:\ failed\ *) ;; + *) return 0 ;; + esac + case "$line" in + *'source: none'*|*'source: status-log'*|*'state: unknown'*|*'occupancy unknown'*) return 0 ;; + esac + outcome=${line#state: } + outcome=${outcome%% *} + case "$outcome" in done|failed) ;; *) return 0 ;; esac + source=$(printf '%s\n' "$line" | sed -n 's/.*source: \([^ ·]*\).*/\1/p') + [ -n "$source" ] && [ "$source" != none ] || return 0 + snapshot=$(single_line "$line") + INC=$(read_incarnation "$meta" "$id") || return 0 + window=$(meta_value_unique "$meta" window) || return 0 + if backend=$(meta_value_unique "$meta" backend 2>/dev/null); then + : + else + state_rc=$? + [ "$state_rc" = 1 ] && backend=tmux || return 0 + fi + case "$backend" in tmux|herdr) ;; *) return 0 ;; esac + if [ "$source" = run-step ]; then + run_step_incarnation_evidence_valid "$id" "$INC" "$outcome" "$snapshot" || return 0 + elif ! fm_pane_idle_proof_valid "$STATE" "$meta" "$id" "$window" "$backend" "$INC" "$RECONCILE_SECS"; then + return 0 + fi + ID=$id + OUTCOME=$outcome + SNAPSHOT=$snapshot + SOURCE=$source + KIND=${kind:-ship} + is_secondmate_home + route_rc=$? + case "$route_rc" in + 0|1) ;; + *) return 0 ;; + esac + if [ "$route_rc" = 0 ]; then + KIND=secondmate + fm_pending_reply_secondmate_route_validate "$FM_HOME" || return 75 + parent_corr=$FM_PENDING_ROUTE_CORR + else + FM_PENDING_ROUTE_CORR= + parent_corr= + fi + FP=$(terminal_outcome_fingerprint "$id" "$INC" "$outcome" "$snapshot" "$KIND" "$parent_corr" "$source") || return 1 + surface_status=0 + terminal_outcome_surfaced "$id" "$meta" "$outcome" "inactive-outcome:$FP" \ + || surface_status=$? + case "$surface_status" in + 0) return 0 ;; + 1) ;; + *) return 2 ;; + esac + if [ "$route_rc" = 0 ]; then + key="inactive-outcome:$FP" + replay_surface_retry_write "$id" "$meta" "$snapshot" "$INC" "$key" 2 || return 1 + publish_secondmate_receipt_and_wake "$meta" "$id" "$window" "$backend" "$INC" "$source" \ + || { publication_status=$?; return "$publication_status"; } + if replay_receipt_exists; then + replay_surface_marker "$id" "$meta" "$snapshot" "$INC" "$key" || return 1 + fi + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + return 0 + fi + if receipt_existing_core; then + key="inactive-outcome:$FP" + replay_surface_retry_write "$id" "$meta" "$snapshot" "$INC" "$key" 2 || return 1 + if [ "$RECEIPT_EXISTING_SUFFIX" = pending ]; then + republish_existing_receipt_wake "$meta" "$id" "$window" "$backend" "$INC" "$source" \ + || { publication_status=$?; return "$publication_status"; } + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + fi + replay_surface_marker "$id" "$meta" "$snapshot" "$INC" "$key" || return 1 + return 0 + else + existing_rc=$? + [ "$existing_rc" = 1 ] || return 1 + fi + key="inactive-outcome:$FP" + replay_surface_retry_write "$id" "$meta" "$snapshot" "$INC" "$key" 2 || return 1 + publish_receipt_and_wake "$meta" "$id" "$window" "$backend" "$INC" "$source" \ + || { publication_status=$?; return "$publication_status"; } + replay_surface_marker "$id" "$meta" "$snapshot" "$INC" "$key" || return 1 + if [ "$FM_WAKE_APPEND_CREATED" = 1 ]; then + printf 'queued inactive outcome: task=%s state=%s fingerprint=%s\n' "$id" "$outcome" "$FP" + fi + return 0 +} + +ack_receipt() { # + local key=$1 row=${2:-} owner_required=${3:-1} expected_generation=${4:-} + local fp rec id kind incarnation outcome snapshot expected_fp parent_task_id parent_home parent_status corr line target existing existing_kind existing_corr claim_state + [ -n "$row" ] || return 2 + case "$owner_required" in 0|1) ;; *) return 2 ;; esac + [ "$owner_required" = 0 ] || drain_claim_owner "$row" || return 2 + case "$key" in inactive-outcome:*) fp=${key#inactive-outcome:} ;; *) return 0 ;; esac + case "$fp" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + claim_state=$(claim_validate "$(claim_path "$fp")" "$fp" "$row") || return 2 + [ "$claim_state" = presented ] || return 2 + [ "$(claim_binary_field "$(claim_path "$fp")" output_started 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$(claim_path "$fp")" output_emitted 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$(claim_path "$fp")" output_complete 2>/dev/null || true)" = 1 ] || return 2 + [ "$(claim_binary_field "$(claim_path "$fp")" output_confirmed 2>/dev/null || true)" = 1 ] || return 2 + if [ "$(claim_field "$(claim_path "$fp")" defer_ack 2>/dev/null || true)" = 1 ]; then + [ "$(claim_binary_field "$(claim_path "$fp")" caller_confirmed 2>/dev/null || true)" = 1 ] || return 2 + fi + if [ "$owner_required" = 0 ]; then + claim_validate_caller_owner "$(claim_path "$fp")" "$expected_generation" || return 2 + fi + rec=$(receipt_path "$fp" pending) + [ ! -L "$rec" ] || return 2 + if [ ! -e "$rec" ]; then + for existing in "$(receipt_path "$fp" presented)" "$(receipt_path "$fp" reported)"; do + [ ! -L "$existing" ] || return 2 + if [ -e "$existing" ]; then + [ -f "$existing" ] || return 2 + prepare_pending_receipt "$existing" || return 2 + existing_kind=$KIND + if [ "$existing_kind" = secondmate ]; then + reported_secondmate_receipt_valid "$existing" || return 2 + existing_corr=$(receipt_field "$existing" parent_corr) + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$existing_corr" \ + "$(receipt_field "$existing" parent_task_id)" \ + "$(receipt_field "$existing" parent_home)" \ + "$(receipt_field "$existing" parent_status)" || return 2 + fi + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 + fm_wake_remove_key_locked "$key" || return 2 + return 1 + fi + done + return 2 + fi + [ -f "$rec" ] || return 2 + prepare_pending_receipt "$rec" || return 2 + id=$(receipt_field "$rec" task_id) + kind=$(receipt_field "$rec" kind) + incarnation=$(receipt_field "$rec" incarnation) + parent_task_id=$(receipt_field "$rec" parent_task_id) + corr=$(receipt_field "$rec" parent_corr) + outcome=$(receipt_field "$rec" outcome) + snapshot=$(receipt_field "$rec" terminal_snapshot) + expected_fp=$(terminal_outcome_fingerprint "$id" "$incarnation" "$outcome" "$snapshot" "$kind" "$corr" "$SOURCE") || return 2 + [ "$expected_fp" = "$fp" ] || return 2 + case "$kind" in ship|scout|secondmate) ;; *) return 2 ;; esac + if [ "$kind" = secondmate ]; then + parent_home=$(receipt_field "$rec" parent_home) + parent_status=$(receipt_field "$rec" parent_status) + corr=$(receipt_field "$rec" parent_corr) + [ -n "$parent_task_id" ] || return 2 + secondmate_ack_report "$FM_HOME" "$parent_task_id" "$parent_home" \ + "$parent_status" "$corr" "$outcome" "$id" "$fp" || return 2 + target=$(receipt_path "$fp" reported) + else + target=$(receipt_path "$fp" presented) + fi + [ ! -L "$target" ] || return 2 + if [ -e "$target" ]; then + [ -f "$target" ] || return 2 + [ "$(receipt_field "$target" fingerprint)" = "$fp" ] || return 2 + if [ "$kind" = secondmate ]; then + reported_secondmate_receipt_valid "$target" || return 2 + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" \ + "$parent_task_id" "$parent_home" "$parent_status" || return 2 + fi + rm -f "$rec" || return 2 + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 + fm_wake_remove_key_locked "$key" || return 2 + return 1 + fi + fm_nofollow_rename "$rec" "$target" 1 || return 2 + if [ "$kind" = secondmate ]; then + fm_pending_reply_secondmate_route_clear_reported "$FM_HOME" "$corr" \ + "$parent_task_id" "$parent_home" "$parent_status" || return 2 + fi + claim_remove "$key" "$row" "$owner_required" "$expected_generation" || return 2 + fm_wake_remove_key_locked "$key" || return 2 + return 0 +} + +confirm_receipt() { + local status=0 caller_pid=${PPID:-} + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 + claim_mark_confirmed "$1" "$2" 0 "$caller_pid" || status=$? + if [ "$status" = 0 ]; then + ack_receipt "$1" "$2" 0 "$caller_pid" || status=$? + fi + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 + return "$status" +} + +caller_output_complete() { + local status=0 caller_pid=${3:-${PPID:-}} + fm_lock_acquire_wait "$FM_WAKE_QUEUE_LOCK" || return 2 + claim_mark_output_complete "$1" "$2" 0 "$caller_pid" || status=$? + fm_lock_release "$FM_WAKE_QUEUE_LOCK" || status=2 + return "$status" +} + +secondmate_ack_report() { # + local secondmate_home=$1 parent_task_id=$2 parent_home=$3 parent_status=$4 corr=$5 outcome=$6 task_id=$7 fp=$8 + local parent_state token route_lock route_marker route_history line phase rc=0 route_lock_held=0 marker_present=0 report_recorded=0 report_probe_status + fm_pending_reply_secondmate_receipt_validate \ + "$secondmate_home" "$parent_task_id" "$parent_home" "$parent_status" "$corr" || return 2 + parent_state=${FM_PENDING_ROUTE_STATE:-} + [ -n "$parent_state" ] || return 2 + fm_pending_reply_txn_lock_acquire "$parent_state" "$corr" token || return 2 + if ! fm_pending_reply_secondmate_receipt_validate \ + "$secondmate_home" "$parent_task_id" "$parent_home" "$parent_status" "$corr"; then + rc=2 + else + phase=$FM_PENDING_ROUTE_PHASE + line="$outcome [corr=$corr]: inactive terminal outcome replayed: task=$task_id fingerprint=$fp" + [ ! -L "$parent_status" ] || rc=2 + if [ "$rc" = 0 ] && [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || rc=2 + fi + if [ "$rc" = 0 ] && [ -f "$parent_status" ]; then + parent_status_line_status "$parent_status" "$line" + report_probe_status=$? + case "$report_probe_status" in + 0) report_recorded=1 ;; + 1) ;; + *) rc=2 ;; + esac + fi + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if [ "$rc" = 0 ] && fm_lock_acquire_wait "$route_lock"; then + route_lock_held=1 + elif [ "$rc" = 0 ]; then + rc=2 + fi + if [ "$route_lock_held" = 1 ]; then + route_marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + route_history=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ -e "$route_marker" ] || [ -L "$route_marker" ] \ + || [ -e "$route_history" ] || [ -L "$route_history" ]; then + marker_present=1 + fi + if [ "$marker_present" = 1 ]; then + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" || rc=2 + elif [ "$phase" != resolved ] \ + && [ "$phase" != retired ] \ + && [ "$report_recorded" != 1 ]; then + rc=2 + fi + if [ "$rc" = 0 ] \ + && [ "$phase" != resolved ] \ + && [ "$phase" != retired ] \ + && [ "$report_recorded" != 1 ]; then + if [ ! -e "$parent_status" ] && [ ! -L "$parent_status" ]; then + parent_status_append_line "$parent_status" "$line" || rc=2 + [ "$rc" = 0 ] && report_recorded=1 + elif [ "$rc" = 0 ]; then + parent_status_line_status "$parent_status" "$line" + report_probe_status=$? + case "$report_probe_status" in + 0) report_recorded=1 ;; + 1) parent_status_append_line "$parent_status" "$line" || rc=2 + [ "$rc" = 0 ] && report_recorded=1 ;; + *) rc=2 ;; + esac + fi + fi + fm_lock_release "$route_lock" || rc=2 + route_lock_held=0 + fi + fi + if [ "$route_lock_held" = 1 ]; then + fm_lock_release "$route_lock" || rc=2 + fi + fm_pending_reply_txn_lock_release "$parent_state" "$corr" "$token" || rc=2 + return "$rc" +} + +scan_locked() { + local startup=${1:-0} marker_mtime now age cursor meta id started=1 cursor_seen=1 + local cursor_identity current_cursor_identity + local scan_started scan_deadline maintenance_deadline remaining rc complete=1 scan_failed=0 find_tmp maintenance_status=0 + local maintenance_started maintenance_phase maintenance_next_phase maintenance_phase_tmp + local maintenance_order next_order maintenance_order_tmp maintenance_ran=0 maintenance_deferred=0 direct_deferred=0 + local find_pending_source=0 find_retain=0 find_ordered=1 batch_consumed=0 batch_complete=1 retry_path= + local find_source_drained=0 + local find_pending_offset=0 find_tmp_owned=0 pending_skip=0 find_enum_cursor= + local find_enum_complete=0 find_enum_persisted=0 child_remaining child_scan_deadline + local pane_idle_index_dir='' + inactive_state_preflight || return 1 + inactive_merge_txn_recover || return 1 + marker_mtime=$(file_mtime "$SCAN_MARKER" 2>/dev/null || true) + now=$(date +%s) + if [ "$startup" != 1 ] && [ -n "$marker_mtime" ]; then + age=$((now - marker_mtime)) + [ "$age" -ge "$RECONCILE_SECS" ] || return 0 + fi + run_maintenance() { + maintenance_phase=$(cat "$MAINTENANCE_PHASE_CURSOR" 2>/dev/null || true) + case "$maintenance_phase" in pending|reported) ;; *) maintenance_phase=pending ;; esac + maintenance_next_phase=$maintenance_phase + MAINTENANCE_ENUM_DEFERRED=0 + MAINTENANCE_ENUM_FAILED=0 + MAINTENANCE_RETRYABLE=0 + maintenance_started=$(clock_millis) + if [ "$maintenance_started" -ge "$scan_deadline" ]; then + maintenance_deferred=1 + return 0 + fi + maintenance_deadline=$((maintenance_started + MAINTENANCE_TURN_SECS * 1000)) + [ "$maintenance_deadline" -gt "$scan_deadline" ] && maintenance_deadline=$scan_deadline + if [ "$maintenance_phase" = pending ]; then + if ! republish_pending_receipts "$maintenance_deadline"; then + if [ "$MAINTENANCE_RETRYABLE" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_DEFERRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_DEADLINE_EXPIRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_FAILED" = 1 ]; then + maintenance_status=1 + elif [ "$(clock_millis)" -ge "$scan_deadline" ]; then + maintenance_deferred=1 + else + maintenance_status=1 + fi + fi + if [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ] \ + && [ "$MAINTENANCE_ITEMS_PROCESSED" = 0 ]; then + if ! repair_reported_secondmate_routes "$maintenance_deadline"; then + if [ "$MAINTENANCE_ENUM_DEFERRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_RETRYABLE" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_DEADLINE_EXPIRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_FAILED" = 1 ]; then + maintenance_status=1 + elif [ "$(clock_millis)" -ge "$scan_deadline" ]; then + maintenance_deferred=1 + else + maintenance_status=1 + fi + fi + if [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ]; then + maintenance_next_phase=pending + fi + elif [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ]; then + maintenance_next_phase=reported + fi + else + if ! repair_reported_secondmate_routes "$maintenance_deadline"; then + if [ "$MAINTENANCE_ENUM_DEFERRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_RETRYABLE" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_DEADLINE_EXPIRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_FAILED" = 1 ]; then + maintenance_status=1 + elif [ "$(clock_millis)" -ge "$scan_deadline" ]; then + maintenance_deferred=1 + else + maintenance_status=1 + fi + fi + if [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ] \ + && [ "$MAINTENANCE_ITEMS_PROCESSED" = 0 ]; then + if ! republish_pending_receipts "$maintenance_deadline"; then + if [ "$MAINTENANCE_RETRYABLE" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_DEFERRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_DEADLINE_EXPIRED" = 1 ]; then + maintenance_deferred=1 + elif [ "$MAINTENANCE_ENUM_FAILED" = 1 ]; then + maintenance_status=1 + elif [ "$(clock_millis)" -ge "$scan_deadline" ]; then + maintenance_deferred=1 + else + maintenance_status=1 + fi + fi + if [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ]; then + maintenance_next_phase=reported + fi + elif [ "$maintenance_status" = 0 ] && [ "$maintenance_deferred" = 0 ]; then + maintenance_next_phase=pending + fi + fi + maintenance_phase_tmp=$(mktemp "$STATE/.inactive-outcome-maintenance.cursor.XXXXXX") || { + maintenance_status=1 + return 0 + } + if [ ! -f "$maintenance_phase_tmp" ] || [ -L "$maintenance_phase_tmp" ] \ + || ! printf '%s\n' "$maintenance_next_phase" | fm_nofollow_write "$maintenance_phase_tmp" \ + || ! fm_nofollow_rename "$maintenance_phase_tmp" "$MAINTENANCE_PHASE_CURSOR"; then + maintenance_status=1 + rm -f "$maintenance_phase_tmp" + fi + } + maintenance_order=$(cat "$MAINTENANCE_ORDER_CURSOR" 2>/dev/null || true) + case "$maintenance_order" in direct|maintenance) ;; *) maintenance_order=direct ;; esac + if [ "$maintenance_order" = direct ]; then + next_order=maintenance + else + next_order=direct + fi + maintenance_order_tmp=$(mktemp "$STATE/.inactive-outcome-maintenance-order.XXXXXX") || return 1 + if [ ! -f "$maintenance_order_tmp" ] || [ -L "$maintenance_order_tmp" ] \ + || ! printf '%s\n' "$next_order" | fm_nofollow_write "$maintenance_order_tmp" \ + || ! fm_nofollow_rename "$maintenance_order_tmp" "$MAINTENANCE_ORDER_CURSOR"; then + rm -f "$maintenance_order_tmp" + return 1 + fi + scan_started=$(clock_millis) + scan_deadline=$((scan_started + DIRECT_SCAN_BUDGET_SECS * 1000)) + if [ "$maintenance_order" = maintenance ]; then + maintenance_ran=1 + run_maintenance + fi + cursor=$(cat "$SCAN_CURSOR" 2>/dev/null || true) + cursor_identity=$(cat "$SCAN_CURSOR_IDENTITY" 2>/dev/null || true) + case "$cursor" in + '') cursor_identity=;; + *[!A-Za-z0-9._-]*) cursor=; cursor_identity=;; + esac + case "$cursor_identity" in + ''|*[!0123456789abcdefABCDEF]*) cursor=; cursor_identity=;; + esac + if [ -n "$cursor_identity" ] && [ "${#cursor_identity}" != 64 ]; then + cursor= + cursor_identity= + fi + if [ -n "$cursor" ] && { [ ! -f "$STATE/$cursor.meta" ] || [ -L "$STATE/$cursor.meta" ]; }; then + cursor= + cursor_identity= + fi + if [ -n "$cursor" ]; then + current_cursor_identity=$(metadata_fingerprint "$STATE/$cursor.meta" 2>/dev/null || true) + if [ -z "$current_cursor_identity" ] || [ "$current_cursor_identity" != "$cursor_identity" ]; then + cursor= + cursor_identity= + fi + fi + if [ -e "$DIRECT_FIND_INCOMPLETE" ] || [ -L "$DIRECT_FIND_INCOMPLETE" ]; then + [ -f "$DIRECT_FIND_INCOMPLETE" ] && [ ! -L "$DIRECT_FIND_INCOMPLETE" ] || return 1 + fi + if [ -e "$DIRECT_FIND_ENUM_CURSOR" ] || [ -L "$DIRECT_FIND_ENUM_CURSOR" ]; then + [ -f "$DIRECT_FIND_ENUM_CURSOR" ] && [ ! -L "$DIRECT_FIND_ENUM_CURSOR" ] || return 1 + find_enum_cursor=$(inactive_text_cursor_read "$DIRECT_FIND_ENUM_CURSOR") || return 1 + case "$find_enum_cursor" in + ''|0) find_enum_cursor=;; + *[!0-9]*) find_enum_cursor=;; + esac + fi + if [ -n "$cursor" ]; then started=0; fi + [ -n "$cursor" ] && cursor_seen=0 + if [ ! -e "$DIRECT_FIND_PENDING" ] && [ ! -L "$DIRECT_FIND_PENDING" ] \ + && inactive_pending_retry_exists "$DIRECT_FIND_RETRY"; then + inactive_pending_retry_restore "$DIRECT_FIND_RETRY" "$DIRECT_FIND_PENDING" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + inactive_pending_offset_write "$DIRECT_FIND_OFFSET" 0 || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + fi + if [ -e "$DIRECT_FIND_PENDING" ] || [ -L "$DIRECT_FIND_PENDING" ]; then + [ -f "$DIRECT_FIND_PENDING" ] && [ ! -L "$DIRECT_FIND_PENDING" ] || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + if [ -s "$DIRECT_FIND_PENDING" ]; then + inactive_pending_merge_retry "$DIRECT_FIND_PENDING" "$DIRECT_FIND_RETRY" "$scan_deadline" + rc=$? + case "$rc" in + 0) ;; + 124) direct_deferred=1 ;; + *) + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + ;; + esac + find_tmp="$DIRECT_FIND_PENDING" + find_pending_source=1 + find_retain=1 + find_pending_offset=$(inactive_pending_offset "$DIRECT_FIND_OFFSET") || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + pending_skip=$find_pending_offset + else + inactive_pending_clear "$DIRECT_FIND_PENDING" "$DIRECT_FIND_OFFSET" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + fi + elif [ -e "$DIRECT_FIND_OFFSET" ] || [ -L "$DIRECT_FIND_OFFSET" ]; then + inactive_pending_clear "$DIRECT_FIND_PENDING" "$DIRECT_FIND_OFFSET" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + fi + if [ "$find_pending_source" = 0 ]; then + find_tmp=$(mktemp "$STATE/.inactive-outcome-find.XXXXXX") || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ] || { + rm -f "$find_tmp" + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + find_tmp_owned=1 + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + direct_deferred=1 + find_retain=1 + else + if inactive_find_candidates "$remaining" "$find_enum_cursor" \ + "$DIRECT_FIND_ENUM_CURSOR" | fm_nofollow_write "$find_tmp"; then + rm -f "$DIRECT_FIND_INCOMPLETE" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + find_enum_complete=1 + if inactive_persist_nul_suffix "$find_tmp" "$DIRECT_FIND_PENDING" 0 "" "$scan_deadline"; then + find_enum_persisted=1 + find_tmp="$DIRECT_FIND_PENDING" + find_tmp_owned=0 + find_pending_source=1 + find_pending_offset=0 + pending_skip=0 + find_retain=1 + remaining=$(budget_remaining_secs "$scan_deadline") + else + rc=$? + [ "$rc" -ne 0 ] || rc=1 + scan_failed=1 + remaining=0 + fi + if [ "$scan_failed" = 0 ] && [ "$remaining" -le 0 ]; then + direct_deferred=1 + find_retain=1 + fi + else + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + find_retain=1 + if inactive_persist_nul_suffix "$find_tmp" "$DIRECT_FIND_PENDING" 0 "" "$scan_deadline"; then + find_enum_persisted=1 + find_tmp="$DIRECT_FIND_PENDING" + find_tmp_owned=0 + find_pending_source=1 + elif [ -f "$find_tmp" ] && [ ! -L "$find_tmp" ]; then + inactive_pending_retry_defer "$find_tmp" "${DIRECT_FIND_PENDING}.retry" "" 1 || true + [ -e "$find_tmp" ] || find_tmp_owned=0 + fi + if [ "$rc" = 124 ]; then + direct_deferred=1 + inactive_text_cursor_write "$DIRECT_FIND_INCOMPLETE" "" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + else + scan_failed=1 + inactive_enum_cursor_discard "$DIRECT_FIND_ENUM_CURSOR" || { + [ "$maintenance_ran" = 1 ] || run_maintenance + return 1 + } + fi + fi + fi + fi + if [ "$find_pending_source" = 1 ]; then + cursor= + started=1 + cursor_seen=1 + batch_consumed=$find_pending_offset + fi + remaining=$(budget_remaining_secs "$scan_deadline") + # At the minimum one-second budget, spend the available time on the direct + # child scan. The indexed metadata pass is an optimization; running it first + # can consume the entire minimum budget before the authoritative state read. + if [ "$remaining" -gt 0 ] && [ "$DIRECT_SCAN_BUDGET_SECS" -gt 1 ]; then + pane_idle_index_dir=$PANE_IDLE_INDEX_DIR + if { [ -d "$pane_idle_index_dir" ] && [ ! -L "$pane_idle_index_dir" ]; } \ + || { [ ! -e "$pane_idle_index_dir" ] && mkdir -p "$pane_idle_index_dir"; }; then + : + else + pane_idle_index_dir= + fi + if [ -n "$pane_idle_index_dir" ] \ + && fm_pane_idle_meta_index_persist "$STATE" "$pane_idle_index_dir" "$scan_deadline"; then + : + else + pane_idle_index_dir= + complete=0 + direct_deferred=1 + find_retain=1 + fi + else + complete=0 + direct_deferred=1 + find_retain=1 + fi + while [ "$scan_failed" = 0 ] && [ "$find_ordered" = 1 ] && IFS= read -r -d '' meta; do + if [ "$pending_skip" -gt 0 ]; then + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + complete=0 + direct_deferred=1 + find_retain=1 + batch_complete=0 + break + fi + pending_skip=$((pending_skip - 1)) + continue + fi + remaining=$(budget_remaining_secs "$scan_deadline") + if [ "$remaining" -le 0 ]; then + complete=0 + direct_deferred=1 + find_retain=1 + batch_complete=0 + break + fi + batch_consumed=$((batch_consumed + 1)) + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + if [ "$started" = 0 ] && [ "$meta" = "$STATE/$cursor.meta" ]; then + cursor= + started=1 + cursor_seen=1 + complete=0 + direct_deferred=1 + find_retain=1 + fi + continue + fi + id=$(basename "$meta" .meta) + valid_task_id "$id" || continue + if [ "$started" = 0 ]; then + if [ ! -f "$STATE/$cursor.meta" ] || [ -L "$STATE/$cursor.meta" ]; then + cursor= + started=1 + cursor_seen=1 + complete=0 + direct_deferred=1 + find_retain=1 + else + [ "$id" = "$cursor" ] || continue + current_cursor_identity=$(metadata_fingerprint "$STATE/$cursor.meta" 2>/dev/null || true) + if [ -n "$cursor_identity" ] && [ "$current_cursor_identity" != "$cursor_identity" ]; then + cursor= + cursor_identity= + started=1 + cursor_seen=1 + complete=0 + direct_deferred=1 + find_retain=1 + else + started=1 + cursor_seen=1 + continue + fi + fi + fi + rc=0 + child_remaining=$remaining + child_scan_deadline=$scan_deadline + if [ "$DIRECT_SCAN_BUDGET_SECS" -eq 1 ]; then + # A one-second minimum must still cover the child shell, lock proof, and + # durable receipt write. Keep this grace bounded and local to the child; + # the parent scan deadline still controls progress and cursor advancement. + child_remaining=$((child_remaining + 1)) + child_scan_deadline=$((child_scan_deadline + 1000)) + fi + ( + export FM_LOCK_WAIT_SECS="$child_remaining" + export FM_INACTIVE_OUTCOME_SCAN_REMAINING_SECS="$child_remaining" + export FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS="$child_scan_deadline" + export FM_INACTIVE_OUTCOME_CHILD_BOUND=1 + export FM_PANE_IDLE_META_INDEX_DIR="${pane_idle_index_dir:-}" + run_bounded_child "$child_remaining" "$SCRIPT_DIR/fm-inactive-reconcile.sh" _child "$id" + ) || rc=$? + if [ "$rc" -ne 0 ]; then + complete=0 + case "$rc" in + 75|124) + direct_deferred=1 + find_retain=1 + if [ -z "$retry_path" ]; then + retry_path=$DIRECT_FIND_RETRY + inactive_retry_file_prepare "$retry_path" || { + scan_failed=1 + break + } + fi + inactive_append_nul_value "$retry_path" "$meta" "$scan_deadline" + append_rc=$? + if [ "$append_rc" = 124 ]; then + direct_deferred=1 + batch_complete=0 + break + elif [ "$append_rc" -ne 0 ]; then + scan_failed=1 + batch_complete=0 + break + fi + continue + ;; + *) + scan_failed=1 + batch_complete=0 + ;; + esac + [ "$scan_failed" = 0 ] || break + continue + fi + current_cursor_identity=$(metadata_fingerprint "$meta" 2>/dev/null || true) + if [ -n "$current_cursor_identity" ] \ + && scan_cursor_write "$id" "$current_cursor_identity"; then + cursor_identity=$current_cursor_identity + : + else + rc=$? + complete=0 + scan_failed=1 + break + fi + done < "$find_tmp" + if [ "$find_pending_source" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$scan_failed" = 0 ] && [ -e "$DIRECT_FIND_INCOMPLETE" ]; then + inactive_pending_clear "$DIRECT_FIND_PENDING" "$DIRECT_FIND_OFFSET" || { + complete=0 + scan_failed=1 + } + find_source_drained=1 + direct_deferred=1 + fi + if [ "$find_pending_source" = 1 ] && [ "$batch_complete" = 1 ] \ + && [ "$scan_failed" = 0 ]; then + find_retain=0 + fi + if [ "$direct_deferred" = 1 ] && [ "$scan_failed" = 0 ]; then + find_retain=1 + fi + if [ "$find_source_drained" = 1 ]; then + : + elif [ "$find_retain" = 1 ] && [ "$scan_failed" = 0 ]; then + inactive_persist_nul_suffix "$find_tmp" "$DIRECT_FIND_PENDING" "$batch_consumed" "" "$scan_deadline" || { + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + if [ "$rc" = 124 ]; then + direct_deferred=1 + else + scan_failed=1 + fi + } + elif [ "$find_pending_source" = 1 ] && [ "$scan_failed" = 0 ] \ + && [ "$batch_complete" = 1 ]; then + inactive_pending_clear "$DIRECT_FIND_PENDING" "$DIRECT_FIND_OFFSET" || { + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + scan_failed=1 + } + fi + if [ "$scan_failed" = 0 ] && [ "$find_enum_persisted" = 1 ]; then + inactive_enum_cursor_promote "$DIRECT_FIND_ENUM_CURSOR" || scan_failed=1 + if [ "$scan_failed" = 0 ] && [ "$find_enum_complete" = 1 ]; then + rm -f "$DIRECT_FIND_ENUM_CURSOR" || scan_failed=1 + fi + elif [ "$scan_failed" != 0 ]; then + inactive_enum_cursor_discard "$DIRECT_FIND_ENUM_CURSOR" || scan_failed=1 + fi + if [ "$find_tmp_owned" = 1 ]; then + if ! rm -f "$find_tmp"; then + rc=$? + [ "$rc" -ne 0 ] || rc=1 + complete=0 + scan_failed=1 + fi + fi + if [ "$maintenance_ran" = 0 ]; then + run_maintenance + fi + if [ "$scan_failed" = 1 ]; then + [ "$rc" -ne 0 ] || rc=1 + return "$rc" + fi + [ "$maintenance_status" = 0 ] || return 1 + if [ "$maintenance_deferred" = 1 ] || [ "$direct_deferred" = 1 ]; then + return 0 + fi + [ "$complete" = 1 ] || return 1 + if [ "$cursor_seen" = 0 ]; then + if [ -n "$cursor" ] && { [ ! -f "$STATE/$cursor.meta" ] || [ -L "$STATE/$cursor.meta" ]; }; then + return 0 + fi + if [ -n "$cursor" ] && [ -n "$cursor_identity" ]; then + current_cursor_identity=$(metadata_fingerprint "$STATE/$cursor.meta" 2>/dev/null || true) + [ "$current_cursor_identity" != "$cursor_identity" ] && return 0 + fi + return 1 + fi + if [ "$complete" = 1 ]; then + rm -f "$SCAN_CURSOR" "$SCAN_CURSOR_IDENTITY" || return 1 + fi + date +%s | fm_nofollow_write "$SCAN_MARKER" || return 1 + return 0 +} + +scan() { + local startup=${1:-0} rc + scan_abort() { + local status=$1 + fm_lock_release "$SCAN_LOCK" || true + trap - EXIT INT TERM + exit "$status" + } + inactive_state_preflight || return 1 + fm_lock_acquire_wait "$SCAN_LOCK" || return 1 + trap 'fm_lock_release "$SCAN_LOCK" || true' EXIT + trap 'scan_abort 130' INT + trap 'scan_abort 143' TERM + scan_locked "$startup" + rc=$? + fm_lock_release "$SCAN_LOCK" || true + trap - EXIT INT TERM + return "$rc" +} + +case "${1:-}" in + scan) + if [ "${2:-}" = --startup ]; then + scan 1 + else + scan 0 + fi + ;; + ack) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + ack_receipt "$2" "$3" "${4:-1}" "${5:-}" + ;; + confirm) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + confirm_receipt "$2" "$3" + ;; + claim) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_reserve "$2" "$3" + ;; + presenting) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_presenting "$2" "$3" + ;; + output-started) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_started "$2" "$3" + ;; + output-emitted) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_emitted "$2" "$3" + ;; + output-emitted-state) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_output_emitted "$2" "$3" + ;; + output-confirmed) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_confirmed "$2" "$3" + ;; + output-complete) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_complete "$2" "$3" + ;; + caller-output-complete) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + caller_output_complete "$2" "$3" "${4:-}" + ;; + caller-output-complete-locked) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_output_complete "$2" "$3" 0 "${4:-}" + ;; + presented) + [ -n "${2:-}" ] && [ -n "${3:-}" ] || exit 2 + claim_mark_presented "$2" "$3" + ;; + _child) + [ -n "${2:-}" ] || exit 2 + reconcile_child "$2" + ;; + _pane-idle) + [ -n "${8:-}" ] || exit 2 + unset FM_PANE_IDLE_META_INDEX_DIR + fm_pane_idle_proof_valid "$2" "$3" "$4" "$5" "$6" "$7" "$8" + ;; + *) + echo "usage: fm-inactive-reconcile.sh scan [--startup] | ack " >&2 + exit 2 + ;; +esac diff --git a/bin/fm-pane-idle-lib.sh b/bin/fm-pane-idle-lib.sh new file mode 100644 index 00000000000..3ed6a82ac1a --- /dev/null +++ b/bin/fm-pane-idle-lib.sh @@ -0,0 +1,1944 @@ +#!/usr/bin/env bash + +_FM_PANE_IDLE_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-safe-io-lib.sh +. "$_FM_PANE_IDLE_LIB_DIR/fm-safe-io-lib.sh" + +FM_PANE_IDLE_BUSY_REGEX_DEFAULT='esc (to )?interrupt|Working\.\.\.|Ctrl\+c:cancel' +FM_PANE_IDLE_META_INDEX_STATE= +FM_PANE_IDLE_META_INDEX_STATE_STAMP= +FM_PANE_IDLE_META_INDEX_SNAPSHOT= +FM_PANE_IDLE_META_INDEX_BUILT=0 + +fm_pane_idle_meta_value_unique() { # + awk -F= -v wanted="$2" ' + $1 == wanted { count++; value=substr($0, index($0, "=") + 1) } + END { + if (count == 1) { print value; exit 0 } + if (count == 0) exit 1 + exit 2 + } + ' "$1" 2>/dev/null +} + +fm_pane_idle_now_ms() { + if command -v clock_millis >/dev/null 2>&1; then + clock_millis + else + printf '%s000' "$(date +%s)" + fi +} + +fm_pane_idle_budget_secs() { + local deadline_ms=$1 now + case "$deadline_ms" in ''|*[!0-9]*) printf '300'; return 0 ;; esac + now=$(fm_pane_idle_now_ms) + [ "$deadline_ms" -gt "$now" ] || return 1 + printf '%s' "$(((deadline_ms - now + 999) / 1000))" +} + +fm_pane_idle_run_bounded_child() { + local seconds=$1 + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + elif command -v perl >/dev/null 2>&1; then + perl -e 'my $t = shift; my $pid = fork; die "fork failed" unless defined $pid; if (!$pid) { setpgrp(0, 0); exec @ARGV or exit 127 } local $SIG{ALRM} = sub { kill "TERM", -$pid; select undef, undef, undef, 0.2; kill "KILL", -$pid; exit 124 }; alarm $t; waitpid $pid, 0; my $status = $?; exit(($status & 127) ? 128 + ($status & 127) : ($status >> 8))' "$seconds" "$@" + else + return 125 + fi +} + +fm_pane_idle_run_bounded_perl() { + local deadline_ms=$1 remaining + shift + if [ -n "$deadline_ms" ]; then + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || return 124 + fm_pane_idle_run_bounded_child "$remaining" perl - "$@" + else + perl - "$@" + fi +} + +fm_pane_idle_compare_files() { + local left=$1 right=$2 deadline_ms=${3:-} remaining + [ -f "$left" ] && [ ! -L "$left" ] || return 1 + [ -f "$right" ] && [ ! -L "$right" ] || return 1 + if [ -n "$deadline_ms" ]; then + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || return 124 + fm_pane_idle_run_bounded_child "$remaining" cmp -s "$left" "$right" + else + cmp -s "$left" "$right" + fi +} + +fm_pane_idle_meta_freshness_path() { + printf '%s/.pane-idle-meta-index/.metadata-freshness' "$1" +} + +fm_pane_idle_meta_freshness_bump() { + local state=$1 boundary_dir boundary_path tmp value + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + boundary_dir="$state/.pane-idle-meta-index" + if [ -e "$boundary_dir" ] || [ -L "$boundary_dir" ]; then + [ -d "$boundary_dir" ] && [ ! -L "$boundary_dir" ] || return 1 + else + mkdir "$boundary_dir" || return 1 + fi + boundary_path="$boundary_dir/.metadata-freshness" + [ ! -L "$boundary_path" ] || return 1 + value="$(fm_pane_idle_now_ms).$$.${RANDOM:-0}" + tmp=$(mktemp "$boundary_path.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + printf '%s\n' "$value" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + [ ! -L "$boundary_path" ] && fm_nofollow_rename "$tmp" "$boundary_path" || { + rm -f "$tmp" + return 1 + } +} + +fm_pane_idle_path_stamp() { + local path=$1 deadline_ms=${2:-} boundary boundary_value + [ -d "$path" ] && [ ! -L "$path" ] || return 1 + boundary=$(fm_pane_idle_meta_freshness_path "$path") + if [ -f "$boundary" ] && [ ! -L "$boundary" ]; then + boundary_value=$(head -n 1 "$boundary" 2>/dev/null || true) + case "$boundary_value" in + ''|*[!A-Za-z0-9._-]*) ;; + *) printf '%s' "$boundary_value"; return 0 ;; + esac + fi + fm_pane_idle_run_bounded_perl "$deadline_ms" "$path" <<'PERL' +use strict; +use warnings; +use Digest::SHA qw(sha256_hex); +use Time::HiRes qw(stat); + +my $state = shift @ARGV; +my @info = stat($state) or exit 1; +print sha256_hex(join("\0", @info[0, 1, 2, 7, 9, 10, 11])); +PERL +} + +fm_pane_idle_meta_index_entries_build() { + local state=$1 entries_path=$2 complete_path=$3 entries_stamp_path=$4 deadline_ms=${5:-} + local partial_path="$entries_path.partial" cookie_path="$entries_path.cookie" + local source_stamp_path="$entries_path.source-stamp" remaining rc + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + [ ! -L "$entries_path" ] && [ ! -L "$complete_path" ] || return 1 + [ ! -L "$entries_stamp_path" ] && [ ! -L "$partial_path" ] || return 1 + [ ! -L "$cookie_path" ] && [ ! -L "$source_stamp_path" ] || return 1 + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || return 124 + fm_pane_idle_run_bounded_child "$remaining" perl - "$state" "$entries_path" \ + "$complete_path" "$entries_stamp_path" "$partial_path" "$cookie_path" \ + "$source_stamp_path" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); +use Digest::SHA qw(sha256_hex); +use Time::HiRes qw(stat); + +my ($state, $entries, $complete, $stamp_path, $partial, $cookie_path, $source_stamp) = @ARGV; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +my $boundary = "$state/.pane-idle-meta-index/.metadata-freshness"; +sub atomic_write { + my ($path, $value) = @_; + return 0 if -l $path; + my $tmp = "$path.tmp.$$"; + my $fh; + sysopen($fh, $tmp, O_WRONLY | O_CREAT | O_EXCL | $nofollow, 0600) or return 0; + binmode($fh); + return 0 unless print($fh $value) && close($fh); + return 0 if -l $path; + rename($tmp, $path) or return 0; + return 1; +} +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path; + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} +sub dir_stamp { + if (-f $boundary && !-l $boundary) { + open(my $bfh, '<', $boundary) or return undef; + local $/; + my $value = <$bfh> // ''; + close($bfh) or return undef; + chomp $value; + return $value if $value =~ /\A[A-Za-z0-9._-]+\z/; + } + my @info = stat($state) or return undef; + return sha256_hex(join("\0", @info[0, 1, 2, 7, 9, 10, 11])); +} +sub read_text { + my ($path) = @_; + return '' unless -e $path; + return undef if -l $path || !-f $path; + open(my $fh, '<', $path) or return undef; + local $/; + my $value = <$fh> // ''; + close($fh) or return undef; + chomp $value; + return $value; +} +my $start = dir_stamp(); +defined($start) or exit 1; +my $old = read_text($source_stamp); +defined($old) or exit 1 if -e $source_stamp; +if (!defined($old) || $old ne $start) { + unlink($partial, $cookie_path, $entries, $complete, $stamp_path); + atomic_write($source_stamp, "$start\n") or exit 1; +} +my $cookie = read_text($cookie_path); +defined($cookie) or exit 1 if -e $cookie_path; +$cookie = '' unless defined($cookie) && ($cookie eq 'EOF' || $cookie =~ /^\d+\z/); +if ($cookie ne 'EOF') { + my $pfh = open_append($partial) or exit 1; + select((select($pfh), $| = 1)[0]); + opendir(my $dh, $state) or exit 1; + seekdir($dh, 0 + $cookie) or exit 1 if $cookie ne ''; + while (defined(my $entry = readdir($dh))) { + my $next = telldir($dh); + defined($next) or exit 1; + if ($entry =~ /\.meta\z/ && $entry !~ /[\r\n]/) { + my $path = "$state/$entry"; + if (-f $path && !-l $path) { + print($pfh $path, "\n") or exit 1; + } + } + atomic_write($cookie_path, "$next\n") or exit 1; + } + closedir($dh) or exit 1; + close($pfh) or exit 1; + atomic_write($cookie_path, "EOF\n") or exit 1; + $cookie = 'EOF'; +} +my $finish = dir_stamp(); +defined($finish) or exit 1; +if ($finish ne $start) { + unlink($partial, $cookie_path, $entries, $complete, $stamp_path); + atomic_write($source_stamp, "$finish\n") or exit 1; + exit 75; +} +open(my $pfh, '<', $partial) or exit 1; +my %seen; +my @paths; +while (defined(my $path = <$pfh>)) { + chomp $path; + next unless $path =~ /\A\Q$state\E\/[^\/\r\n]+\.meta\z/; + next unless -f $path && !-l $path; + next if $seen{$path}++; + push @paths, $path; +} +close($pfh) or exit 1; +@paths = sort @paths; +atomic_write($entries, join('', map { "$_\n" } @paths)) or exit 1; +atomic_write($stamp_path, "$finish\n") or exit 1; +atomic_write($complete, "complete\n") or exit 1; +unlink($partial, $cookie_path, $source_stamp) or exit 1; +PERL + rc=$? + [ "$rc" = 75 ] && return 124 + [ "$rc" -ne 0 ] || return 0 + return "$rc" +} + +fm_pane_idle_meta_index_collect() { + local state=$1 output=$2 deadline_ms=${3:-} worker rc remaining sorted_tmp state_stamp entries_stamp post_state_stamp + local progress_dir cursor_path records_path complete_path seen_path aggregate_path aggregate_cursor_path + local aggregate_complete_path sorted_path sorted_complete_path entries_path entries_complete_path entries_stamp_path + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + [ ! -L "$output" ] || return 1 + case "$output" in "$state"/*) ;; *) return 1 ;; esac + progress_dir="$state/.pane-idle-meta-index" + if [ -e "$progress_dir" ] || [ -L "$progress_dir" ]; then + [ -d "$progress_dir" ] && [ ! -L "$progress_dir" ] || return 1 + else + mkdir "$progress_dir" || return 1 + fi + cursor_path="$progress_dir/.scan.cursor" + records_path="$progress_dir/.scan.records" + complete_path="$progress_dir/.scan.complete" + seen_path="$progress_dir/.scan.seen" + aggregate_path="$progress_dir/.scan.aggregate" + aggregate_cursor_path="$progress_dir/.scan.aggregate.cursor" + aggregate_complete_path="$progress_dir/.scan.aggregate.complete" + sorted_path="$progress_dir/.scan.sorted" + sorted_complete_path="$progress_dir/.scan.sorted.complete" + entries_path="$progress_dir/.scan.entries" + entries_complete_path="$progress_dir/.scan.entries.complete" + entries_stamp_path="$progress_dir/.scan.entries.stamp" + for path in "$cursor_path" "$records_path" "$complete_path" "$seen_path" \ + "$aggregate_path" "$aggregate_cursor_path" "$aggregate_complete_path" \ + "$sorted_path" "$sorted_complete_path" "$entries_path" \ + "$entries_complete_path" "$entries_stamp_path"; do + if [ -e "$path" ] || [ -L "$path" ]; then + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + fi + done + if [ -e "$complete_path" ]; then + rm -f "$cursor_path" "$records_path" "$seen_path" "$aggregate_path" \ + "$aggregate_cursor_path" "$aggregate_complete_path" "$sorted_path" \ + "$sorted_complete_path" "$complete_path" || return 1 + fi + while :; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + return 124 + fi + state_stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || return $? + entries_stamp= + source_stamp=$(cat "$entries_path.source-stamp" 2>/dev/null || true) + if [ -f "$entries_path" ] && [ ! -L "$entries_path" ] \ + && [ -f "$entries_complete_path" ] && [ ! -L "$entries_complete_path" ] \ + && [ -f "$entries_stamp_path" ] && [ ! -L "$entries_stamp_path" ]; then + entries_stamp=$(cat "$entries_stamp_path" 2>/dev/null || true) + fi + if [ "$entries_stamp" != "$state_stamp" ]; then + if [ "$source_stamp" != "$state_stamp" ]; then + rm -f "$entries_path" "$entries_complete_path" "$entries_stamp_path" \ + "$entries_path.partial" "$entries_path.cookie" "$entries_path.source-stamp" \ + || return 1 + else + rm -f "$entries_path" "$entries_complete_path" "$entries_stamp_path" || return 1 + fi + rm -f \ + "$cursor_path" "$records_path" "$seen_path" "$aggregate_path" \ + "$aggregate_cursor_path" "$aggregate_complete_path" "$sorted_path" \ + "$sorted_complete_path" || return 1 + fm_pane_idle_meta_index_entries_build "$state" "$entries_path" \ + "$entries_complete_path" "$entries_stamp_path" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + fi + break + done + command -v perl >/dev/null 2>&1 || return 125 + perl - "$state" "$entries_path" "$cursor_path" "$records_path" "$complete_path" "$seen_path" "$deadline_ms" <<'PERL' & +use strict; +use warnings; +use Fcntl qw(:DEFAULT); + +my ($state, $entries_path, $cursor_path, $records_path, $complete_path, $seen_path, $deadline, $output) = @ARGV; +$deadline = undef unless defined($deadline) && $deadline =~ /^\d+\z/ && $deadline ne ''; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; + +sub open_read { + my ($path) = @_; + return undef if -l $path || !defined($nofollow); + my $fh; + sysopen($fh, $path, O_RDONLY | $nofollow) or return undef; + return $fh; +} + +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path && defined($nofollow); + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} + +sub write_atomic { + my ($path, $value) = @_; + return 0 if -l $path; + my $tmp = "$path.tmp.$$"; + my $fh; + sysopen($fh, $tmp, O_WRONLY | O_CREAT | O_EXCL, 0600) or return 0; + binmode($fh); + if (!print($fh $value) || !close($fh)) { + close($fh); + return 0; + } + return 0 if -l $path; + rename($tmp, $path) or return 0; + return 1; +} + +sub open_output { + my ($path) = @_; + return undef if -l $path || !-f $path || !defined($nofollow); + my $fh; + sysopen($fh, $path, O_WRONLY | O_TRUNC | $nofollow) or return undef; + return $fh; +} + +sub repair_records_tail { + my ($path) = @_; + return 1 if -l $path || !-f $path; + my $fh = open_read($path) or return 0; + binmode($fh); + local $/ = "\0"; + my $tail_start = 0; + my $partial = 0; + while (1) { + $tail_start = tell($fh); + defined($tail_start) or return 0; + my $meta = <$fh>; + last unless defined($meta); + my $window = <$fh>; + my $count = <$fh>; + if (!defined($window) || !defined($count) + || $meta !~ /\0\z/ || $window !~ /\0\z/ || $count !~ /\0\z/) { + $partial = 1; + last; + } + } + close($fh) or return 0; + return 1 unless $partial; + my $wfh; + sysopen($wfh, $path, O_RDWR | $nofollow) or return 0; + my $ok = truncate($wfh, $tail_start); + $ok = 0 unless close($wfh); + return $ok ? 1 : 0; +} + +sub expired { + return defined($deadline) && int(time() * 1000) >= $deadline; +} + +my $cursor = ''; +if (-e $cursor_path) { + my $cfh = open_read($cursor_path) or exit 1; + local $/; + my $raw_cursor = <$cfh> // ''; + $raw_cursor =~ /\A(.*)\n\z/s or exit 1; + $cursor = $1; + close($cfh) or exit 1; + $cursor ne '' or exit 1; +} +if ($cursor ne '' && $cursor ne 'EOF') { + $cursor =~ /\A\Q$state\E\/[^\/\r\n\x00]+\.meta\z/ or exit 1; + if (-e $cursor || -l $cursor) { + (-f $cursor && !-l $cursor) or exit 1; + } +} + +my %record_keys; +my %seen_paths; +my $state_loaded = 0; +while (1) { + exit 124 if expired(); + exit 1 unless -d $state && !-l $state; + if (!-e $records_path) { + my $rfh = open_append($records_path) or exit 1; + close($rfh) or exit 1; + } + if (!-e $seen_path) { + my $sfh = open_append($seen_path) or exit 1; + close($sfh) or exit 1; + } + exit 0 if $cursor eq 'EOF'; + if (!$state_loaded) { + repair_records_tail($records_path) or exit 1; + my $rfh = open_read($records_path) or exit 1; + binmode($rfh); + { + local $/ = "\0"; + while (defined(my $record_path = <$rfh>)) { + exit 124 if expired(); + my $record_window = <$rfh>; + my $record_count = <$rfh>; + defined($record_window) && defined($record_count) or exit 1; + $record_path =~ s/\0\z// or exit 1; + $record_window =~ s/\0\z// or exit 1; + $record_count =~ s/\0\z// or exit 1; + $record_keys{$record_path . "\0" . $record_window . "\0" . $record_count} = 1; + } + } + close($rfh) or exit 1; + my $sfh = open_read($seen_path) or exit 1; + while (defined(my $seen_path_value = <$sfh>)) { + exit 124 if expired(); + chomp $seen_path_value; + $seen_paths{$seen_path_value} = 1; + } + close($sfh) or exit 1; + $state_loaded = 1; + } + my $efh = open_read($entries_path) or exit 1; + binmode($efh); + while (defined(my $path = <$efh>)) { + exit 124 if expired(); + chomp $path; + next if $path !~ /\.meta\z/ || $path =~ /[\r\n]/; + next if $cursor ne '' && $path le $cursor; + $path =~ /\A\Q$state\E\/[^\/]+\.meta\z/ or exit 1; + if (-f $path && !-l $path) { + my $fh = open_read($path) or exit 1; + my ($window, $window_count) = ('', 0); + while (defined(my $line = <$fh>)) { + exit 124 if expired(); + chomp $line; + next unless $line =~ /^window=(.*)\z/; + $window = $1; + $window_count++; + } + close($fh) or exit 1; + if ($window_count == 1 && $window ne '') { + my $record_key = $path . "\0" . $window . "\0" . "1"; + if (!$record_keys{$record_key}) { + my $rfh = open_append($records_path) or exit 1; + binmode($rfh); + print $rfh $path, "\0", $window, "\0", "1", "\0" or exit 1; + close($rfh) or exit 1; + $record_keys{$record_key} = 1; + } + } + } + if (!$seen_paths{$path}) { + my $sfh = open_append($seen_path) or exit 1; + print $sfh $path, "\n" or exit 1; + close($sfh) or exit 1; + $seen_paths{$path} = 1; + } + write_atomic($cursor_path, "$path\n") or exit 1; + $cursor = $path; + } + close($efh) or exit 1; + write_atomic($cursor_path, "EOF\n") or exit 1; + $cursor = 'EOF'; +} +PERL + worker=$! + while kill -0 "$worker" 2>/dev/null; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + kill -TERM "$worker" 2>/dev/null || true + kill -KILL "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + rm -f "$output" + return 124 + fi + sleep 0.01 + done + wait "$worker" + rc=$? + rm -f "$cursor_path.tmp.$worker" "$complete_path.tmp.$worker" + if [ "$rc" -ne 0 ]; then + rm -f "$output" + return "$rc" + fi + if [ ! -e "$aggregate_complete_path" ]; then + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || { + rm -f "$output" + return 124 + } + if fm_pane_idle_run_bounded_child "$remaining" perl - "$records_path" \ + "$aggregate_path" "$aggregate_cursor_path" "$aggregate_complete_path" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); + +my ($records_path, $aggregate_path, $cursor_path, $complete_path) = @ARGV; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +sub open_read { + my ($path) = @_; + return undef if -l $path || !-f $path; + my $fh; + sysopen($fh, $path, O_RDONLY | $nofollow) or return undef; + return $fh; +} +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path; + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} +sub write_atomic { + my ($path, $value) = @_; + return 0 if -l $path; + my $tmp = "$path.tmp.$$"; + my $fh; + sysopen($fh, $tmp, O_WRONLY | O_CREAT | O_EXCL, 0600) or return 0; + binmode($fh); + return 0 unless print($fh $value) && close($fh); + return 0 if -l $path; + rename($tmp, $path) or return 0; + return 1; +} +sub repair_records_tail { + my ($path) = @_; + return 1 if -l $path || !-f $path; + my $fh = open_read($path) or return 0; + binmode($fh); + local $/ = "\0"; + my $tail_start = 0; + my $partial = 0; + while (1) { + $tail_start = tell($fh); + defined($tail_start) or return 0; + my $meta = <$fh>; + last unless defined($meta); + my $window = <$fh>; + my $count = <$fh>; + if (!defined($window) || !defined($count) + || $meta !~ /\0\z/ || $window !~ /\0\z/ || $count !~ /\0\z/) { + $partial = 1; + last; + } + } + close($fh) or return 0; + return 1 unless $partial; + my $wfh; + sysopen($wfh, $path, O_RDWR | $nofollow) or return 0; + my $ok = truncate($wfh, $tail_start); + $ok = 0 unless close($wfh); + return $ok ? 1 : 0; +} +sub repair_newline_tail { + my ($path) = @_; + return 1 if !-e $path; + return 0 if -l $path || !-f $path; + my $fh; + sysopen($fh, $path, O_RDWR | $nofollow) or return 0; + binmode($fh); + seek($fh, 0, 2) or return 0; + my $size = tell($fh); + defined($size) or return 0; + if ($size > 0) { + seek($fh, $size - 1, 0) or return 0; + my $last = getc($fh); + return 0 unless defined($last); + if ($last ne "\n") { + my $position = $size; + my $boundary = 0; + while ($position > 0) { + my $start = $position > 4096 ? $position - 4096 : 0; + my $length = $position - $start; + seek($fh, $start, 0) or return 0; + my $buffer = ''; + read($fh, $buffer, $length) == $length or return 0; + for (my $index = length($buffer) - 1; $index >= 0; $index--) { + if (substr($buffer, $index, 1) eq "\n") { + $boundary = $start + $index + 1; + last; + } + } + last if $boundary; + $position = $start; + } + truncate($fh, $boundary) or return 0; + } + } + close($fh) or return 0; + return 1; +} +repair_records_tail($records_path) or exit 1; +repair_newline_tail($aggregate_path) or exit 1; +my $offset = 0; +if (-e $cursor_path) { + my $cfh = open_read($cursor_path) or exit 1; + my $value = <$cfh> // ''; + close($cfh) or exit 1; + chomp $value; + $value =~ /^\d+\z/ or exit 1; + $offset = 0 + $value; +} +my $rfh = open_read($records_path) or exit 1; +seek($rfh, $offset, 0) or exit 1; +my %aggregate_lines; +if (-e $aggregate_path) { + my $existing_afh = open_read($aggregate_path) or exit 1; + while (defined(my $line = <$existing_afh>)) { + chomp $line; + $aggregate_lines{$line} = 1; + } + close($existing_afh) or exit 1; +} +my $afh = open_append($aggregate_path) or exit 1; +binmode($rfh); +binmode($afh); +local $/ = "\0"; +while (defined(my $meta = <$rfh>)) { + my $window = <$rfh>; + my $count = <$rfh>; + last unless defined($window) && defined($count); + $meta =~ s/\0\z// or exit 1; + $window =~ s/\0\z// or exit 1; + $count =~ s/\0\z// or exit 1; + $window =~ /^\S+\z/ or exit 1; + $count =~ /^\d+\z/ or exit 1; + my $line = unpack('H*', $window) . "\t" . unpack('H*', $meta); + if (!$aggregate_lines{$line}) { + print $afh $line, "\n" or exit 1; + $aggregate_lines{$line} = 1; + } + my $position = tell($rfh); + defined($position) or exit 1; + write_atomic($cursor_path, "$position\n") or exit 1; +} +close($rfh) or exit 1; +close($afh) or exit 1; +write_atomic($complete_path, "complete\n") or exit 1; +PERL + then : + else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$output" + return "$rc" + fi + fi + if [ ! -e "$sorted_complete_path" ]; then + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || { + rm -f "$output" + return 124 + } + sorted_tmp=$(mktemp "$sorted_path.XXXXXX") || { + rm -f "$output" + return 1 + } + [ -f "$sorted_tmp" ] && [ ! -L "$sorted_tmp" ] || { + rm -f "$sorted_tmp" "$output" + return 1 + } + if fm_pane_idle_run_bounded_child "$remaining" env LC_ALL=C sort -u \ + "$aggregate_path" | fm_nofollow_write "$sorted_tmp" \ + && fm_nofollow_rename "$sorted_tmp" "$sorted_path" \ + && fm_pane_idle_meta_index_cursor_write "$sorted_complete_path" complete; then + : + else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$sorted_tmp" "$output" + return "$rc" + fi + fi + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || { + rm -f "$output" + return 124 + } + if fm_pane_idle_run_bounded_child "$remaining" perl - "$sorted_path" "$output" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); + +my ($sorted_path, $output) = @ARGV; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +open(my $sfh, '<', $sorted_path) or exit 1; +binmode($sfh); +my $ofh; +sysopen($ofh, $output, O_WRONLY | O_TRUNC | $nofollow) or exit 1; +binmode($ofh); +my ($window, $first_meta, $count) = ('', '', 0); +sub flush_window { + return unless $count; + print $ofh $first_meta, "\0", $window, "\0", $count, "\0" or exit 1; +} +while (defined(my $line = <$sfh>)) { + chomp $line; + my ($window_hex, $meta_hex) = split(/\t/, $line, 2); + exit 1 unless defined($window_hex) && defined($meta_hex) + && $window_hex =~ /\A(?:[0-9A-Fa-f]{2})*\z/ + && $meta_hex =~ /\A(?:[0-9A-Fa-f]{2})*\z/; + my $current_window = pack('H*', $window_hex); + my $current_meta = pack('H*', $meta_hex); + if ($current_window ne $window) { + flush_window(); + $window = $current_window; + $first_meta = $current_meta; + $count = 1; + } else { + $count++; + } +} +close($sfh) or exit 1; +flush_window(); +close($ofh) or exit 1; +PERL + then : + else + rc=$? + [ "$rc" -ne 0 ] || rc=124 + rm -f "$output" + return "$rc" + fi + post_state_stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || { + rc=$? + rm -f "$output" + return "$rc" + } + if [ "$post_state_stamp" != "$state_stamp" ]; then + rm -f "$output" "$entries_path" "$entries_complete_path" "$entries_stamp_path" \ + "$cursor_path" "$records_path" "$seen_path" "$aggregate_path" \ + "$aggregate_cursor_path" "$aggregate_complete_path" "$sorted_path" \ + "$sorted_complete_path" "$complete_path" + return 124 + fi + fm_pane_idle_meta_index_cursor_write "$complete_path" complete || { + rm -f "$output" + return 1 + } + for path in "$output" "$records_path" "$complete_path" "$seen_path" \ + "$aggregate_path" "$aggregate_complete_path" "$sorted_path" \ + "$sorted_complete_path"; do + [ -f "$path" ] && [ ! -L "$path" ] || { + rm -f "$output" + return 1 + } + done + if [ -e "$cursor_path" ] || [ -L "$cursor_path" ]; then + [ -f "$cursor_path" ] && [ ! -L "$cursor_path" ] || { + rm -f "$output" + return 1 + } + fi +} + +fm_pane_idle_meta_index_build() { # [deadline-ms] [force] + local state=$1 deadline_ms=${2:-} force=${3:-} rc tmp stamp post_stamp snapshot progress_dir + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + progress_dir="$state/.pane-idle-meta-index" + if [ -e "$progress_dir" ] || [ -L "$progress_dir" ]; then + [ -d "$progress_dir" ] && [ ! -L "$progress_dir" ] || return 1 + else + mkdir "$progress_dir" || return 1 + fi + stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || return $? + snapshot="$progress_dir/snapshot" + if [ -e "$snapshot" ] || [ -L "$snapshot" ]; then + [ -f "$snapshot" ] && [ ! -L "$snapshot" ] || return 1 + fi + if [ "$FM_PANE_IDLE_META_INDEX_STATE" = "$state" ] \ + && [ "$force" != force ] \ + && [ "$FM_PANE_IDLE_META_INDEX_BUILT" = 1 ] \ + && [ "$FM_PANE_IDLE_META_INDEX_STATE_STAMP" = "$stamp" ] \ + && [ -f "$snapshot" ]; then + return 0 + fi + FM_PANE_IDLE_META_INDEX_BUILT=0 + FM_PANE_IDLE_META_INDEX_SNAPSHOT= + while :; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + return 124 + fi + stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || return $? + tmp=$(mktemp "$snapshot.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + fm_pane_idle_meta_index_collect "$state" "$tmp" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "$tmp" + return "$rc" + fi + post_stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || { + rc=$? + rm -f "$tmp" + return "$rc" + } + if [ "$post_stamp" != "$stamp" ]; then + rm -f "$tmp" + continue + fi + fm_nofollow_rename "$tmp" "$snapshot" || { + rm -f "$tmp" + return 1 + } + FM_PANE_IDLE_META_INDEX_STATE=$state + FM_PANE_IDLE_META_INDEX_STATE_STAMP=$post_stamp + FM_PANE_IDLE_META_INDEX_SNAPSHOT=$snapshot + FM_PANE_IDLE_META_INDEX_BUILT=1 + return 0 + done +} + +fm_pane_idle_meta_index_lookup_snapshot() { + local state=$1 snapshot=$2 window=$3 deadline_ms=${4:-} candidate rc + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + [ -f "$snapshot" ] && [ ! -L "$snapshot" ] || return 1 + candidate=$(fm_pane_idle_run_bounded_perl "$deadline_ms" "$snapshot" "$window" <<'PERL' +use strict; +use warnings; +my ($snapshot, $wanted) = @ARGV; +open(my $fh, '<', $snapshot) or exit 2; +binmode($fh); +local $/ = "\0"; +my ($candidate, $matches) = ('', 0); +while (1) { + my $meta = <$fh>; + last unless defined $meta; + my $window = <$fh>; + my $count = <$fh>; + defined($window) && defined($count) or exit 2; + chomp($meta, $window, $count); + if ($window eq $wanted) { + $count eq '1' or exit 1; + $matches++; + $matches == 1 or exit 1; + $candidate = $meta; + } +} +close($fh) or exit 2; +$matches == 1 or exit 1; +print $candidate or exit 2; +PERL + ) + rc=$? + [ "$rc" = 0 ] || return "$rc" + case "$candidate" in "$state"/*) ;; *) return 1 ;; esac + [ -f "$candidate" ] && [ ! -L "$candidate" ] || return 1 + printf '%s' "$candidate" +} + +fm_pane_idle_meta_index_windows_from_snapshot() { + local snapshot=$1 deadline_ms=${2:-} + [ -f "$snapshot" ] && [ ! -L "$snapshot" ] || return 1 + fm_pane_idle_run_bounded_perl "$deadline_ms" "$snapshot" <<'PERL' +use strict; +use warnings; +my $snapshot = shift @ARGV; +open(my $fh, '<', $snapshot) or exit 2; +binmode($fh); +local $/ = "\0"; +my %seen; +while (1) { + my $meta = <$fh>; + last unless defined $meta; + my $window = <$fh>; + my $count = <$fh>; + defined($window) && defined($count) or exit 2; + chomp($meta, $window, $count); + next if $window =~ /[\r\n]/; + next if exists $seen{$window}; + $seen{$window} = 1; + print $window, "\n" or exit 2; +} +close($fh) or exit 2; +PERL +} + +fm_pane_idle_meta_index_windows_from_snapshot_resumable() { + local snapshot=$1 cursor=${2:-0} deadline_ms=${3:-} + [ -f "$snapshot" ] && [ ! -L "$snapshot" ] || return 1 + case "$cursor" in ''|*[!0-9]*) return 1 ;; esac + fm_pane_idle_run_bounded_perl "$deadline_ms" "$snapshot" "$cursor" <<'PERL' +use strict; +use warnings; +my ($snapshot, $cursor) = @ARGV; +open(my $fh, '<', $snapshot) or exit 2; +binmode($fh); +seek($fh, $cursor, 0) or exit 2; +local $/ = "\0"; +my %seen; +while (1) { + my $meta = <$fh>; + last unless defined $meta; + my $window = <$fh>; + my $count = <$fh>; + defined($window) && defined($count) or exit 2; + $meta =~ s/\0\z// or exit 2; + $window =~ s/\0\z// or exit 2; + $count =~ s/\0\z// or exit 2; + my $next = tell($fh); + defined($next) or exit 2; + if ($count ne '1' || $window =~ /[\r\n\t]/ || exists $seen{$window}) { + print $next, "\0\0\0" or exit 2; + next; + } + $seen{$window} = 1; + print $next, "\0", $window, "\0", $meta, "\0" or exit 2; +} +close($fh) or exit 2; +PERL +} + +fm_pane_idle_meta_index_windows_direct() { + local state=$1 deadline_ms=${2:-} + local entries_path="$state/.pane-idle-meta-index/.scan.entries" + local complete_path="$state/.pane-idle-meta-index/.scan.entries.complete" + [ -f "$entries_path" ] && [ ! -L "$entries_path" ] || return 124 + [ -f "$complete_path" ] && [ ! -L "$complete_path" ] || return 124 + fm_pane_idle_run_bounded_perl "$deadline_ms" "$state" "$entries_path" <<'PERL' +use strict; +use warnings; +my ($state, $entries_path) = @ARGV; +open(my $entries_fh, '<', $entries_path) or exit 2; +my %seen; +while (defined(my $path = <$entries_fh>)) { + chomp $path; + $path =~ /\A\Q$state\E\/[^\/\r\n]+\.meta\z/ or exit 2; + next unless -f $path && !-l $path; + open(my $fh, '<', $path) or exit 2; + my ($window, $count) = ('', 0); + while (defined(my $line = <$fh>)) { + if ($line =~ /\Awindow=(.*)\n\z/) { + $window = $1; + $count++; + } + } + close($fh) or exit 2; + next unless $count == 1 && $window !~ /[\r\n]/; + next if exists $seen{$window}; + $seen{$window} = 1; + print $window, "\n" or exit 2; +} +close($entries_fh) or exit 2; +PERL +} + +fm_pane_idle_meta_index_windows_direct_resumable() { + local state=$1 cursor=${2:-} deadline_ms=${3:-} + local entries_path="$state/.pane-idle-meta-index/.scan.entries" + local complete_path="$state/.pane-idle-meta-index/.scan.entries.complete" + local windows_path="$state/.pane-idle-meta-index/.scan.windows" + local windows_complete_path="$state/.pane-idle-meta-index/.scan.windows.complete" + local windows_partial_path="$state/.pane-idle-meta-index/.scan.windows.partial" + local windows_cursor_path="$state/.pane-idle-meta-index/.scan.windows.cursor" + local windows_stamp_path="$state/.pane-idle-meta-index/.scan.windows.stamp" + local entries_stamp_path="$state/.pane-idle-meta-index/.scan.entries.stamp" + local source_stamp + [ -f "$entries_path" ] && [ ! -L "$entries_path" ] || return 124 + [ -f "$complete_path" ] && [ ! -L "$complete_path" ] || return 124 + if [ -e "$entries_stamp_path" ] || [ -L "$entries_stamp_path" ]; then + [ -f "$entries_stamp_path" ] && [ ! -L "$entries_stamp_path" ] || return 124 + fi + local sidecar + for sidecar in "$windows_path" "$windows_complete_path" "$windows_partial_path" \ + "$windows_cursor_path" "$windows_stamp_path"; do + if [ -e "$sidecar" ] || [ -L "$sidecar" ]; then + [ -f "$sidecar" ] && [ ! -L "$sidecar" ] || return 124 + fi + done + case "$cursor" in ''|0) cursor=0;; *[!0-9]*) return 1 ;; esac + source_stamp=$(cat "$entries_stamp_path" 2>/dev/null || true) + [ -n "$source_stamp" ] || source_stamp=$(fm_pane_idle_path_stamp "$state") || return 1 + fm_pane_idle_run_bounded_perl "$deadline_ms" "$state" "$entries_path" "$cursor" "$windows_path" \ + "$windows_complete_path" "$windows_partial_path" "$windows_cursor_path" \ + "$windows_stamp_path" "$source_stamp" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); +my ($state, $entries_path, $cursor, $windows_path, $windows_complete_path, + $windows_partial_path, $windows_cursor_path, $windows_stamp_path, $source_stamp) = @ARGV; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 2; +sub atomic_write { + my ($path, $value) = @_; + return 0 if -l $path; + my $tmp = "$path.tmp.$$"; + my $nofollow = eval { Fcntl::O_NOFOLLOW() }; + defined($nofollow) or return 0; + sysopen(my $fh, $tmp, O_WRONLY | O_CREAT | O_EXCL | $nofollow, 0600) or return 0; + binmode($fh); + if (!print($fh $value) || !close($fh)) { + close($fh); + unlink($tmp); + return 0; + } + return 0 if -l $path; + if (!rename($tmp, $path)) { + unlink($tmp); + return 0; + } + return 1; +} +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path; + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} +sub read_text { + my ($path) = @_; + return '' unless -e $path; + return undef if -l $path || !-f $path; + open(my $fh, '<', $path) or return undef; + local $/; + my $value = <$fh> // ''; + close($fh) or return undef; + chomp $value; + return $value; +} +my $stored_stamp = read_text($windows_stamp_path); +defined($stored_stamp) or exit 2 if -e $windows_stamp_path; +if (!defined($stored_stamp) || $stored_stamp ne $source_stamp) { + unlink($windows_path, $windows_complete_path, $windows_partial_path, $windows_cursor_path); + atomic_write($windows_stamp_path, "$source_stamp\n") or exit 2; +} +if (!-e $windows_complete_path) { + my $raw_cursor = read_text($windows_cursor_path); + defined($raw_cursor) or exit 2 if -e $windows_cursor_path; + $raw_cursor = '0' unless defined($raw_cursor) && $raw_cursor =~ /^\d+\z/; + my $efh; + open($efh, '<', $entries_path) or exit 2; + seek($efh, 0 + $raw_cursor, 0) or exit 2; + my $pfh = open_append($windows_partial_path) or exit 2; + while (defined(my $path = <$efh>)) { + my $next = tell($efh); + defined($next) or exit 2; + chomp $path; + if ($path =~ /\A\Q$state\E\/[^\/\r\n]+\.meta\z/) { + -l $path and exit 2; + if (-f $path) { + open(my $mfh, '<', $path) or exit 2; + my ($window, $count) = ('', 0); + while (defined(my $line = <$mfh>)) { + if ($line =~ /\Awindow=(.*)\n\z/) { + $window = $1; + $count++; + } + } + close($mfh) or exit 2; + if ($count == 1 && $window !~ /[\r\n\t]/) { + print($pfh $window, "\n") or exit 2; + } + } + } + atomic_write($windows_cursor_path, "$next\n") or exit 2; + } + close($efh) or exit 2; + close($pfh) or exit 2; + my %counts; + open(my $rfh, '<', $windows_partial_path) or exit 2; + while (defined(my $window = <$rfh>)) { + chomp $window; + $counts{$window}++ if $window ne ''; + } + close($rfh) or exit 2; + my $content = ''; + for my $window (sort keys %counts) { + $content .= $window . "\t" . $counts{$window} . "\n"; + } + atomic_write($windows_path, $content) or exit 2; + atomic_write($windows_complete_path, "complete\n") or exit 2; + unlink($windows_partial_path, $windows_cursor_path) or exit 2; +} +my %counts; +open(my $wfh, '<', $windows_path) or exit 2; +while (defined(my $line = <$wfh>)) { + chomp $line; + my ($window, $count) = split(/\t/, $line, 2); + defined($count) && $count =~ /^\d+\z/ or exit 2; + $counts{$window} = $count; +} +close($wfh) or exit 2; +open(my $efh, '<', $entries_path) or exit 2; +seek($efh, 0 + $cursor, 0) or exit 2; +while (defined(my $path = <$efh>)) { + my $next = tell($efh); + defined($next) or exit 2; + chomp $path; + if ($path !~ /\A\Q$state\E\/[^\/\r\n]+\.meta\z/) { + print $next, "\0\0\0" or exit 2; + next; + } + -l $path and exit 2; + if (!-f $path) { + print $next, "\0\0\0" or exit 2; + next; + } + open(my $mfh, '<', $path) or exit 2; + my ($window, $count) = ('', 0); + while (defined(my $line = <$mfh>)) { + if ($line =~ /\Awindow=(.*)\n\z/) { + $window = $1; + $count++; + } + } + close($mfh) or exit 2; + if ($count != 1 || $window =~ /[\r\n\t]/ || ($counts{$window} // 0) != 1) { + print $next, "\0\0\0" or exit 2; + next; + } + print $next, "\0", $window, "\0", $path, "\0" or exit 2; +} +close($efh) or exit 2; +PERL +} + +fm_pane_idle_meta_for_window() { # [deadline-ms] + local state=$1 window=$2 deadline_ms=${3:-} rc + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + fm_pane_idle_meta_index_build "$state" "$deadline_ms" + rc=$? + [ "$rc" = 0 ] || return "$rc" + fm_pane_idle_meta_index_lookup_snapshot "$state" \ + "$FM_PANE_IDLE_META_INDEX_SNAPSHOT" "$window" "$deadline_ms" +} + +fm_pane_idle_meta_for_window_bounded() { # + local state=$1 window=$2 deadline_ms=$3 meta rc + meta=$(fm_pane_idle_meta_for_window "$state" "$window" "$deadline_ms") + rc=$? + if [ "$rc" = 0 ]; then + printf '%s' "$meta" + return 0 + fi + [ "$rc" = 124 ] || return "$rc" + fm_pane_idle_meta_for_window_direct "$state" "$window" "$deadline_ms" +} + +fm_pane_idle_meta_index_lookup_entries() { + local state=$1 entries_path=$2 window=$3 deadline_ms=${4:-} candidate rc + [ -f "$entries_path" ] && [ ! -L "$entries_path" ] || return 1 + candidate=$(fm_pane_idle_run_bounded_perl "$deadline_ms" "$state" \ + "$entries_path" "$window" <<'PERL' +use strict; +use warnings; +my ($state, $entries_path, $wanted) = @ARGV; +open(my $entries_fh, '<', $entries_path) or exit 2; +my ($candidate, $matches) = ('', 0); +while (defined(my $path = <$entries_fh>)) { + chomp $path; + $path =~ /\A\Q$state\E\/[^\/\r\n]+\.meta\z/ or exit 2; + next unless -f $path && !-l $path; + open(my $fh, '<', $path) or exit 2; + my ($current, $count) = ('', 0); + while (defined(my $line = <$fh>)) { + chomp $line; + next unless $line =~ /\Awindow=(.*)\z/; + $current = $1; + $count++; + } + close($fh) or exit 2; + next unless $count == 1 && $current eq $wanted; + $matches++; + $matches == 1 or exit 1; + $candidate = $path; +} +close($entries_fh) or exit 2; +$matches == 1 or exit 1; +print $candidate or exit 2; +PERL + ) + rc=$? + [ "$rc" = 0 ] || return "$rc" + case "$candidate" in "$state"/*) ;; *) return 1 ;; esac + [ -f "$candidate" ] && [ ! -L "$candidate" ] || return 1 + printf '%s' "$candidate" +} + +fm_pane_idle_meta_for_window_direct() { # + local state=$1 window=$2 deadline_ms=${3:-${FM_INACTIVE_OUTCOME_SCAN_DEADLINE_MS:-}} + local candidate rc tmp + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + if [ -f "$state/.pane-idle-meta-index/.scan.entries" ] \ + && [ -f "$state/.pane-idle-meta-index/.scan.entries.complete" ]; then + fm_pane_idle_meta_index_lookup_entries "$state" \ + "$state/.pane-idle-meta-index/.scan.entries" "$window" "$deadline_ms" + return $? + fi + tmp=$(mktemp "$state/.pane-idle-meta-direct.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + fm_pane_idle_meta_index_collect "$state" "$tmp" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "$tmp" + return "$rc" + fi + candidate=$(fm_pane_idle_meta_index_lookup_snapshot "$state" "$tmp" "$window" "$deadline_ms") + rc=$? + rm -f "$tmp" + [ "$rc" = 0 ] || return "$rc" + printf '%s' "$candidate" +} + +fm_pane_idle_meta_index_cursor_write() { + local path=$1 value=$2 tmp + [ -n "$value" ] || return 1 + case "$value" in *$'\n'*|*$'\r'*|*$'\t'*) return 1 ;; esac + [ ! -L "$path" ] || return 1 + tmp=$(mktemp "$path.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + if ! printf '%s\n' "$value" | fm_nofollow_write "$tmp" || ! fm_nofollow_rename "$tmp" "$path"; then + rm -f "$tmp" + return 1 + fi +} + +fm_pane_idle_meta_index_persist() { + local state=$1 directory=$2 deadline_ms=${3:-} window meta count key path + local cursor_path ready_path snapshot_path reclaim_cursor_path cursor='' cursor_found=0 started=1 + local reclaim_entries_path reclaim_entries_complete_path snapshot_source + local snapshot_tmp snapshot_changed=1 tmp rc compare_rc source_stamp publish_stamp final_stamp + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + [ -d "$directory" ] && [ ! -L "$directory" ] || return 1 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + fm_pane_idle_meta_index_build "$state" "$deadline_ms" + rc=$? + [ "$rc" = 0 ] || return "$rc" + snapshot_source=$FM_PANE_IDLE_META_INDEX_SNAPSHOT + source_stamp=$FM_PANE_IDLE_META_INDEX_STATE_STAMP + [ -n "$source_stamp" ] || return 1 + [ -f "$snapshot_source" ] && [ ! -L "$snapshot_source" ] || return 1 + cursor_path="$directory/.cursor" + ready_path="$directory/.ready" + snapshot_path="$directory/.snapshot" + reclaim_cursor_path="$directory/.reclaim.cursor" + reclaim_entries_path="$directory/.reclaim.entries" + reclaim_entries_complete_path="$directory/.reclaim.entries.complete" + for path in "$cursor_path" "$ready_path" "$snapshot_path" "$reclaim_cursor_path" \ + "$reclaim_entries_path" "$reclaim_entries_complete_path" \ + "$reclaim_entries_path.partial" "$reclaim_entries_path.cookie"; do + if [ -e "$path" ] || [ -L "$path" ]; then + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + fi + done + if [ -e "$ready_path" ]; then + rm -f "$ready_path" || return 1 + fi + snapshot_tmp=$(mktemp "$snapshot_path.XXXXXX") || return 1 + [ -f "$snapshot_tmp" ] && [ ! -L "$snapshot_tmp" ] || { rm -f "$snapshot_tmp"; return 1; } + if [ -e "$cursor_path" ]; then + [ -f "$cursor_path" ] && [ ! -L "$cursor_path" ] || { + rm -f "$snapshot_tmp" + return 1 + } + cursor=$(cat "$cursor_path" 2>/dev/null || true) + case "$cursor" in *$'\n'*|*$'\r'*|*$'\t'*) + rm -f "$snapshot_tmp" + return 1 + ;; + esac + fi + while :; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + rm -f "$snapshot_tmp" + return 124 + fi + IFS= read -r -d '' meta || break + IFS= read -r -d '' window || { rm -f "$snapshot_tmp"; return 1; } + IFS= read -r -d '' count || { rm -f "$snapshot_tmp"; return 1; } + case "$meta" in "$state"/*) ;; *) rm -f "$snapshot_tmp"; return 1 ;; esac + [ -f "$meta" ] && [ ! -L "$meta" ] || { rm -f "$snapshot_tmp"; return 1; } + printf '%s\0%s\0%s\0' "$meta" "$window" "$count" | fm_nofollow_append "$snapshot_tmp" || { + rm -f "$snapshot_tmp" + return 1 + } + [ -n "$cursor" ] && [ "$meta" = "$cursor" ] && cursor_found=1 + done < "$snapshot_source" + if [ -f "$snapshot_path" ] && [ ! -L "$snapshot_path" ]; then + fm_pane_idle_compare_files "$snapshot_tmp" "$snapshot_path" "$deadline_ms" + compare_rc=$? + case "$compare_rc" in + 0) snapshot_changed=0 ;; + 1) ;; + 124) rm -f "$snapshot_tmp"; return 124 ;; + *) rm -f "$snapshot_tmp"; return 1 ;; + esac + fi + if [ "$snapshot_changed" = 1 ]; then + rm -f "$reclaim_cursor_path" "$reclaim_entries_path" \ + "$reclaim_entries_complete_path" "$reclaim_entries_path.partial" \ + "$reclaim_entries_path.cookie" || { + rm -f "$snapshot_tmp" + return 1 + } + [ ! -L "$snapshot_path" ] || { rm -f "$snapshot_tmp"; return 1; } + fm_nofollow_rename "$snapshot_tmp" "$snapshot_path" || { rm -f "$snapshot_tmp"; return 1; } + snapshot_changed=0 + cursor= + started=1 + fi + if [ "$snapshot_changed" != 1 ] && [ -n "$cursor" ] \ + && [ "$cursor_found" = 0 ]; then + cursor= + started=1 + elif [ "$snapshot_changed" != 1 ] && [ -n "$cursor" ]; then + started=0 + fi + publish_stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || { + rc=$? + rm -f "$snapshot_tmp" + return "$rc" + } + [ "$publish_stamp" = "$source_stamp" ] || { + rm -f "$snapshot_tmp" + return 124 + } + while :; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + rm -f "$snapshot_tmp" + return 124 + fi + IFS= read -r -d '' meta || break + IFS= read -r -d '' window || { rm -f "$snapshot_tmp"; return 1; } + IFS= read -r -d '' count || { rm -f "$snapshot_tmp"; return 1; } + if [ "$started" = 0 ]; then + if [ "$meta" = "$cursor" ]; then + started=1 + fi + continue + fi + key=$(fm_pane_idle_sha256 "$window") || { rm -f "$snapshot_tmp"; return 1; } + path="$directory/$key" + [ ! -L "$path" ] || { rm -f "$snapshot_tmp"; return 1; } + if [ -e "$path" ]; then + [ -f "$path" ] || { rm -f "$snapshot_tmp"; return 1; } + fi + tmp=$(mktemp "$path.XXXXXX") || { rm -f "$snapshot_tmp"; return 1; } + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp" "$snapshot_tmp"; return 1; } + if ! printf '%s\n%s\n' "$count" "$meta" | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$path"; then + rm -f "$tmp" "$snapshot_tmp" + return 1 + fi + fm_pane_idle_meta_index_cursor_write "$cursor_path" "$meta" || { + rm -f "$snapshot_tmp" + return 1 + } + done < "$snapshot_source" + rm -f "$cursor_path" || { rm -f "$snapshot_tmp"; return 1; } + rm -f "$snapshot_tmp" || return 1 + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + return 124 + fi + fm_pane_idle_meta_index_reclaim "$directory" "$deadline_ms" "$snapshot_source" || return $? + final_stamp=$(fm_pane_idle_path_stamp "$state" "$deadline_ms") || return $? + if [ "$final_stamp" != "$source_stamp" ]; then + rm -f "$ready_path" + return 124 + fi + fm_pane_idle_meta_index_cursor_write "$ready_path" ready || return 1 +} + +fm_pane_idle_meta_index_reclaim() { + local directory=$1 deadline_ms=${2:-} snapshot_source=${3:-} current_tmp cursor_path worker rc path key meta window count + local entries_path entries_complete_path remaining + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac + [ -d "$directory" ] && [ ! -L "$directory" ] || return 1 + [ -f "$snapshot_source" ] && [ ! -L "$snapshot_source" ] || return 1 + cursor_path="$directory/.reclaim.cursor" + if [ -e "$cursor_path" ] || [ -L "$cursor_path" ]; then + [ -f "$cursor_path" ] && [ ! -L "$cursor_path" ] || return 1 + fi + entries_path="$directory/.reclaim.entries" + entries_complete_path="$directory/.reclaim.entries.complete" + for path in "$entries_path" "$entries_complete_path"; do + if [ -e "$path" ] || [ -L "$path" ]; then + [ -f "$path" ] && [ ! -L "$path" ] || { + return 1 + } + fi + done + command -v perl >/dev/null 2>&1 || return 125 + current_tmp=$(mktemp "$directory/.reclaim-current.XXXXXX") || return 1 + [ -f "$current_tmp" ] && [ ! -L "$current_tmp" ] || { + rm -f "$current_tmp" + return 1 + } + while :; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + rm -f "$current_tmp" + return 124 + fi + IFS= read -r -d '' meta || break + IFS= read -r -d '' window || { rm -f "$current_tmp"; return 1; } + IFS= read -r -d '' count || { rm -f "$current_tmp"; return 1; } + key=$(fm_pane_idle_sha256 "$window") || { + rm -f "$current_tmp" + return 1 + } + [ "${#key}" = 64 ] || { rm -f "$current_tmp"; return 1; } + case "$key" in *[!0123456789abcdefABCDEF]*) rm -f "$current_tmp"; return 1 ;; esac + printf '%s\n' "$key" | fm_nofollow_append "$current_tmp" || { + rm -f "$current_tmp" + return 1 + } + done < "$snapshot_source" + if [ ! -f "$entries_path" ] || [ ! -f "$entries_complete_path" ]; then + fm_pane_idle_meta_index_reclaim_entries_build \ + "$directory" "$entries_path" "$entries_complete_path" "$deadline_ms" + rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "$current_tmp" + return "$rc" + fi + fi + perl - "$directory" "$current_tmp" "$entries_path" "$cursor_path" "$deadline_ms" <<'PERL' & +use strict; +use warnings; +use Fcntl qw(:DEFAULT); + +my ($directory, $current_path, $entries_path, $cursor_path, $deadline) = @ARGV; +$deadline = undef unless defined($deadline) && $deadline =~ /^\d+\z/ && $deadline ne ''; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +sub expired { + return defined($deadline) && int(time() * 1000) >= $deadline; +} +sub write_progress { + my ($path, $value) = @_; + return 0 if -l $path || !defined($nofollow); + my $fh; + sysopen($fh, $path, O_WRONLY | O_CREAT | O_TRUNC | $nofollow, 0600) or return 0; + binmode($fh); + return 0 unless print($fh $value, "\n") && close($fh); + return 1; +} +open(my $cfh, '<', $current_path) or exit 1; +my %current; +while (defined(my $key = <$cfh>)) { + exit 124 if expired(); + chomp $key; + $current{$key} = 1 if $key =~ /^[0-9A-Fa-f]{64}\z/; +} +close($cfh) or exit 1; +my $cursor = ''; +if (-e $cursor_path) { + exit 1 if -l $cursor_path; + open(my $rfh, '<', $cursor_path) or exit 1; + my $value = <$rfh> // ''; + close($rfh) or exit 1; + chomp $value; + exit 1 if $value ne '' && $value !~ /^(?:\d+|[0-9A-Fa-f]{64}|legacy:\d+:[0-9A-Fa-f]{64})\z/; + $cursor = $value; +} +my $cursor_line = $cursor =~ /^\d+\z/ ? 0 + $cursor : 0; +my $legacy_cursor = ''; +if ($cursor =~ /^legacy:(\d+):([0-9A-Fa-f]{64})\z/) { + $cursor_line = 0 + $1; + $legacy_cursor = $2; +} elsif ($cursor =~ /^[0-9A-Fa-f]{64}\z/) { + $legacy_cursor = $cursor; +} +sub write_cursor { + my ($path, $line, $legacy) = @_; + return write_progress($path, $legacy ne '' ? "legacy:$line:$legacy" : $line); +} +my $efh; +sysopen($efh, $entries_path, O_RDONLY | $nofollow) or exit 1; +my $line_no = 0; +while (defined(my $entry = <$efh>)) { + exit 124 if expired(); + $line_no++; + chomp $entry; + next if $cursor_line > 0 && $line_no <= $cursor_line; + if ($legacy_cursor ne '' && $entry !~ /^[0-9A-Fa-f]{64}\z/) { + write_cursor($cursor_path, $line_no, $legacy_cursor) or exit 1; + next; + } + if ($legacy_cursor ne '' && $entry =~ /^[0-9A-Fa-f]{64}\z/ && $entry le $legacy_cursor) { + write_cursor($cursor_path, $line_no, $legacy_cursor) or exit 1; + next; + } + $legacy_cursor = '' if $legacy_cursor ne '' && $entry =~ /^[0-9A-Fa-f]{64}\z/ && $entry gt $legacy_cursor; + if ($entry =~ /^[0-9A-Fa-f]{64}\z/) { + my $path = "$directory/$entry"; + exit 1 if -l $path; + if (-e $path) { + exit 1 unless -f $path; + if (!exists $current{$entry}) { + unlink($path) or exit 1; + } + } + } + write_cursor($cursor_path, $line_no, '') or exit 1; +} +close($efh) or exit 1; +PERL + worker=$! + while kill -0 "$worker" 2>/dev/null; do + if [ -n "$deadline_ms" ] && [ "$(fm_pane_idle_now_ms)" -ge "$deadline_ms" ]; then + kill -TERM "$worker" 2>/dev/null || true + kill -KILL "$worker" 2>/dev/null || true + wait "$worker" 2>/dev/null || true + rm -f "$current_tmp" + return 124 + fi + sleep 0.01 + done + wait "$worker" + rc=$? + rm -f "$current_tmp" || return 1 + if [ "$rc" -ne 0 ]; then + return "$rc" + fi + rm -f "$cursor_path" || return 1 +} + +fm_pane_idle_meta_index_reclaim_entries_build() { + local directory=$1 entries_path=$2 complete_path=$3 deadline_ms=${4:-} + local partial_path="$entries_path.partial" cookie_path="$entries_path.cookie" remaining rc + [ -d "$directory" ] && [ ! -L "$directory" ] || return 1 + [ ! -L "$entries_path" ] && [ ! -L "$complete_path" ] || return 1 + [ ! -L "$partial_path" ] && [ ! -L "$cookie_path" ] || return 1 + remaining=$(fm_pane_idle_budget_secs "$deadline_ms") || return 124 + fm_pane_idle_run_bounded_child "$remaining" perl - "$directory" "$entries_path" \ + "$complete_path" "$partial_path" "$cookie_path" <<'PERL' +use strict; +use warnings; +use Fcntl qw(:DEFAULT); + +my ($directory, $entries, $complete, $partial, $cookie_path) = @ARGV; +my $nofollow = eval { Fcntl::O_NOFOLLOW() }; +defined($nofollow) or exit 1; +sub atomic_write { + my ($path, $value) = @_; + return 0 if -l $path; + my $tmp = "$path.tmp.$$"; + my $fh; + sysopen($fh, $tmp, O_WRONLY | O_CREAT | O_EXCL | $nofollow, 0600) or return 0; + binmode($fh); + return 0 unless print($fh $value) && close($fh); + return 0 if -l $path; + rename($tmp, $path) or return 0; +} +sub open_append { + my ($path) = @_; + return undef if -l $path; + my $flags = O_WRONLY | O_APPEND; + if (-e $path) { + return undef unless -f $path; + $flags |= $nofollow; + } else { + $flags |= O_CREAT | O_EXCL; + } + my $fh; + sysopen($fh, $path, $flags, 0600) or return undef; + return $fh; +} +sub read_text { + my ($path) = @_; + return '' unless -e $path; + return undef if -l $path || !-f $path; + open(my $fh, '<', $path) or return undef; + local $/; + my $value = <$fh> // ''; + close($fh) or return undef; + chomp $value; + return $value; +} +my $cookie = read_text($cookie_path); +defined($cookie) or exit 1 if -e $cookie_path; +$cookie = '' unless defined($cookie) && ($cookie eq 'EOF' || $cookie =~ /^\d+\z/); +if ($cookie ne 'EOF') { + my $pfh = open_append($partial) or exit 1; + select((select($pfh), $| = 1)[0]); + opendir(my $dh, $directory) or exit 1; + seekdir($dh, 0 + $cookie) or exit 1 if $cookie ne ''; + while (defined(my $entry = readdir($dh))) { + my $next = telldir($dh); + defined($next) or exit 1; + if ($entry =~ /^[0-9A-Fa-f]{64}\z/) { + print($pfh $entry, "\n") or exit 1; + } + atomic_write($cookie_path, "$next\n") or exit 1; + } + closedir($dh) or exit 1; + close($pfh) or exit 1; + atomic_write($cookie_path, "EOF\n") or exit 1; + $cookie = 'EOF'; +} +open(my $pfh, '<', $partial) or exit 1; +my %seen; +my @entries; +while (defined(my $entry = <$pfh>)) { + chomp $entry; + next unless $entry =~ /^[0-9A-Fa-f]{64}\z/; + next if $seen{$entry}++; + push @entries, $entry; +} +close($pfh) or exit 1; +@entries = sort @entries; +atomic_write($entries, join('', map { "$_\n" } @entries)) or exit 1; +atomic_write($complete, "complete\n") or exit 1; +unlink($partial, $cookie_path) or exit 1; +PERL + rc=$? + [ "$rc" -ne 0 ] || return 0 + return "$rc" +} + +fm_pane_idle_meta_for_window_indexed() { + local directory=$1 window=$2 key path count meta current + [ -d "$directory" ] && [ ! -L "$directory" ] || return 1 + key=$(fm_pane_idle_sha256 "$window") || return 1 + path="$directory/$key" + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + { + IFS= read -r count + IFS= read -r meta + } < "$path" || return 1 + [ "$count" = 1 ] || return 1 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + current=$(fm_pane_idle_meta_value_unique "$meta" window) || return 1 + [ "$current" = "$window" ] || return 1 + printf '%s' "$meta" +} + +fm_pane_idle_sha256() { + if command -v shasum >/dev/null 2>&1; then + printf '%s' "$1" | shasum -a 256 | awk '{print $1}' + elif command -v sha256sum >/dev/null 2>&1; then + printf '%s' "$1" | sha256sum | awk '{print $1}' + else + return 1 + fi +} + +fm_pane_idle_hash() { + if command -v md5 >/dev/null 2>&1; then + md5 -q + elif command -v md5sum >/dev/null 2>&1; then + md5sum | awk '{print $1}' + else + return 1 + fi +} + +fm_pane_idle_current_hash() { # + local backend=$1 target=$2 lines=${3:-40} tail native=unknown + command -v fm_backend_capture >/dev/null 2>&1 || return 1 + if [ "$backend" = herdr ] && command -v fm_backend_busy_state >/dev/null 2>&1; then + native=$(FM_BACKEND_HERDR_NO_SERVER_START=1 fm_backend_busy_state "$backend" "$target" 2>/dev/null || printf 'unknown') + case "$native" in + busy) return 1 ;; + idle|unknown) ;; + *) return 1 ;; + esac + fi + if [ "$backend" = herdr ]; then + tail=$(FM_BACKEND_HERDR_NO_SERVER_START=1 fm_backend_capture "$backend" "$target" "$lines" 2>/dev/null) || return 1 + else + tail=$(fm_backend_capture "$backend" "$target" "$lines" 2>/dev/null) || return 1 + fi + if printf '%s' "$tail" | grep -v '^[[:space:]]*$' | tail -6 \ + | grep -qiE "${FM_BUSY_REGEX:-$FM_PANE_IDLE_BUSY_REGEX_DEFAULT}"; then + return 1 + fi + printf '%s' "$tail" | fm_pane_idle_hash +} + +fm_pane_idle_current_matches() { # + local current + current=$(fm_pane_idle_current_hash "$1" "$2" 40) || return 1 + [ "$current" = "$3" ] +} + +fm_pane_idle_read_incarnation() { # + local meta=$1 token tasktmp window worktree seed digest rc + if token=$(fm_pane_idle_meta_value_unique "$meta" spawn_incarnation); then + case "$token" in + ''|legacy-unknown|*[!A-Za-z0-9._:-]*) return 1 ;; + esac + printf '%s' "$token" + return 0 + else + rc=$? + [ "$rc" = 1 ] || return 1 + fi + if tasktmp=$(fm_pane_idle_meta_value_unique "$meta" tasktmp 2>/dev/null); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + tasktmp= + fi + if window=$(fm_pane_idle_meta_value_unique "$meta" window 2>/dev/null); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + window= + fi + if worktree=$(fm_pane_idle_meta_value_unique "$meta" worktree 2>/dev/null); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + worktree= + fi + if [ -n "$tasktmp" ]; then + seed="legacy|tasktmp=$tasktmp" + else + seed="legacy|window=$window|worktree=$worktree" + fi + digest=$(fm_pane_idle_sha256 "$seed") || return 1 + printf 'legacy-%s' "${digest:0:32}" +} + +fm_pane_idle_task_is_ordinary() { # + local kind rc + if kind=$(fm_pane_idle_meta_value_unique "$1" kind); then + : + else + rc=$? + [ "$rc" = 1 ] && kind=ship || return 1 + fi + case "$kind" in ship|scout) return 0 ;; esac + return 1 +} + +fm_pane_idle_field_safe() { + case "$1" in + ''|*$'\n'*|*$'\r'*|*$'\t'*|*=*) return 1 ;; + esac +} + +fm_pane_idle_path() { # + printf '%s/.pane-idle/%s' "$1" "$2" +} + +fm_pane_idle_key() { # + printf '%s' "$1" | tr ':/.' '___' +} + +fm_pane_idle_clear() { # + local state=$1 task=$2 dir path + dir="$state/.pane-idle" + if [ -e "$dir" ] || [ -L "$dir" ]; then + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + fi + path=$(fm_pane_idle_path "$state" "$task") + [ ! -e "$path" ] && return 0 + [ ! -L "$path" ] && [ -f "$path" ] || return 1 + rm -f "$path" +} + +fm_pane_idle_clear_for_window() { # [deadline-ms] + local state=$1 window=$2 deadline_ms=${3:-} meta task + meta=$(fm_pane_idle_meta_for_window "$state" "$window" "$deadline_ms" 2>/dev/null || true) + [ -n "$meta" ] || return 0 + task=${meta##*/} + task=${task%.meta} + fm_pane_idle_clear "$state" "$task" +} + +fm_pane_idle_write() { # + local state=$1 meta=$2 task=$3 window=$4 backend=$5 pane_hash=$6 samples=$7 + local path dir tmp token meta_window current_backend value rc + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + fm_pane_idle_task_is_ordinary "$meta" || return 1 + case "$task" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + for value in "$window" "$backend" "$pane_hash"; do + fm_pane_idle_field_safe "$value" || return 1 + done + case "$backend" in tmux|herdr) ;; *) return 1 ;; esac + case "$pane_hash" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + case "$samples" in ''|*[!0-9]*) return 1 ;; esac + [ "$samples" -ge 2 ] || return 1 + meta_window=$(fm_pane_idle_meta_value_unique "$meta" window) || return 1 + [ "$meta_window" = "$window" ] || return 1 + if current_backend=$(fm_pane_idle_meta_value_unique "$meta" backend 2>/dev/null); then + : + else + rc=$? + [ "$rc" = 1 ] || return 1 + current_backend=tmux + fi + [ "$current_backend" = "$backend" ] || return 1 + token=$(fm_pane_idle_read_incarnation "$meta" "$task") || return 1 + dir="$state/.pane-idle" + [ ! -L "$state" ] && [ -d "$state" ] || return 1 + if [ -e "$dir" ]; then + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + else + mkdir "$dir" || return 1 + fi + path=$(fm_pane_idle_path "$state" "$task") + [ ! -L "$path" ] || return 1 + tmp=$(mktemp "$dir/.tmp.XXXXXX") || return 1 + if printf 'schema=fm-jt-pane-idle.v1\ntask=%s\nwindow=%s\nbackend=%s\nspawn_incarnation=%s\npane_hash=%s\nsample_count=%s\nobserved_epoch=%s\n' "$task" "$window" "$backend" "$token" "$pane_hash" "$samples" "$(date +%s)" | fm_nofollow_write "$tmp" && fm_nofollow_rename "$tmp" "$path"; then + return 0 + fi + rm -f "$tmp" + return 1 +} + +fm_pane_idle_proof_valid() { # + local state=$1 meta=$2 task=$3 window=$4 backend=$5 incarnation=$6 max_age=$7 + local path proof_task proof_window proof_backend proof_incarnation pane_hash samples observed now age + local key hash_file count_file current_hash current_count unique_meta + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + fm_pane_idle_task_is_ordinary "$meta" || return 1 + if [ -n "${FM_PANE_IDLE_META_INDEX_DIR:-}" ]; then + unique_meta=$(fm_pane_idle_meta_for_window_indexed "$FM_PANE_IDLE_META_INDEX_DIR" "$window" 2>/dev/null || true) + else + unique_meta=$(fm_pane_idle_meta_for_window_direct "$state" "$window" 2>/dev/null || true) + fi + [ "$unique_meta" = "$meta" ] || return 1 + path=$(fm_pane_idle_path "$state" "$task") + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["task"]=1; allowed["window"]=1 + allowed["backend"]=1; allowed["spawn_incarnation"]=1; allowed["pane_hash"]=1 + allowed["sample_count"]=1; allowed["observed_epoch"]=1 + required["schema"]=1; required["task"]=1; required["window"]=1 + required["backend"]=1; required["spawn_incarnation"]=1; required["pane_hash"]=1 + required["sample_count"]=1; required["observed_epoch"]=1 + valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + values[key]=substr($0, index($0, "=") + 1) + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !(valid && values["schema"] == "fm-jt-pane-idle.v1") + } + ' "$path" 2>/dev/null || return 1 + proof_task=$(fm_pane_idle_meta_value_unique "$path" task) || return 1 + proof_window=$(fm_pane_idle_meta_value_unique "$path" window) || return 1 + proof_backend=$(fm_pane_idle_meta_value_unique "$path" backend) || return 1 + proof_incarnation=$(fm_pane_idle_meta_value_unique "$path" spawn_incarnation) || return 1 + pane_hash=$(fm_pane_idle_meta_value_unique "$path" pane_hash) || return 1 + samples=$(fm_pane_idle_meta_value_unique "$path" sample_count) || return 1 + observed=$(fm_pane_idle_meta_value_unique "$path" observed_epoch) || return 1 + [ "$proof_task" = "$task" ] && [ "$proof_window" = "$window" ] || return 1 + [ "$proof_backend" = "$backend" ] && [ "$proof_incarnation" = "$incarnation" ] || return 1 + case "$pane_hash" in ''|*[!A-Fa-f0-9]*) return 1 ;; esac + case "$samples" in ''|*[!0-9]*) return 1 ;; esac + [ "$samples" -ge 2 ] || return 1 + case "$observed" in ''|*[!0-9]*) return 1 ;; esac + case "$max_age" in ''|*[!0-9]*) return 1 ;; esac + now=$(date +%s) + age=$((now - observed)) + [ "$age" -ge 0 ] && [ "$age" -le "$max_age" ] || return 1 + key=$(fm_pane_idle_key "$window") + hash_file="$state/.hash-$key" + count_file="$state/.count-$key" + [ -f "$hash_file" ] && [ ! -L "$hash_file" ] || return 1 + [ -f "$count_file" ] && [ ! -L "$count_file" ] || return 1 + current_hash=$(cat "$hash_file" 2>/dev/null || true) + current_count=$(cat "$count_file" 2>/dev/null || true) + [ "$current_hash" = "$pane_hash" ] || return 1 + case "$current_count" in ''|*[!0-9]*) return 1 ;; esac + [ "$current_count" -ge "$samples" ] || return 1 + fm_pane_idle_current_matches "$backend" "$window" "$pane_hash" +} diff --git a/bin/fm-pending-reply-lib.sh b/bin/fm-pending-reply-lib.sh index 230cbd279a6..43db16063e5 100755 --- a/bin/fm-pending-reply-lib.sh +++ b/bin/fm-pending-reply-lib.sh @@ -69,6 +69,8 @@ # shellcheck source=bin/fm-marker-lib.sh _FM_PENDING_REPLY_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/null)" || _FM_PENDING_REPLY_LIB_DIR="." +# shellcheck source=bin/fm-safe-io-lib.sh +. "$_FM_PENDING_REPLY_LIB_DIR/fm-safe-io-lib.sh" # shellcheck source=bin/fm-marker-lib.sh . "$_FM_PENDING_REPLY_LIB_DIR/fm-marker-lib.sh" # shellcheck source=bin/fm-backend.sh @@ -81,6 +83,7 @@ _FM_PENDING_REPLY_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/n FM_PENDING_REPLY_SCHEMA='fm-pending-reply.v1' FM_PENDING_REPLY_CORR_RE='(^|[^[:alnum:]_])corr=([A-Fa-f0-9]{16})($|[^[:alnum:]_])' FM_PENDING_REPLY_GRACE_DEFAULT=120 +FM_PENDING_REPLY_ROUTE_COMMITTED=0 fm_pending_reply_now() { if [ -n "${FM_PENDING_REPLY_NOW:-}" ]; then @@ -131,6 +134,22 @@ fm_pending_reply_source_identity() { # (cd "$1" 2>/dev/null && pwd -P) } +fm_pending_reply_expected_parent_home() { # + local state=$1 state_identity override_identity + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + state_identity=$(fm_pending_reply_source_identity "$state") || return 1 + if [ -n "${FM_STATE_OVERRIDE:-}" ]; then + override_identity=$(fm_pending_reply_source_identity "$FM_STATE_OVERRIDE") || override_identity= + if [ "$override_identity" = "$state_identity" ]; then + [ -n "${FM_HOME:-}" ] || return 1 + fm_pending_reply_source_identity "$FM_HOME" + return + fi + fi + [ "$(basename "$state_identity")" = state ] || return 1 + dirname "$state_identity" +} + fm_pending_reply_source_key() { # printf '%s' "$1" | cksum 2>/dev/null | awk '{printf "%s-%s", $1, $2}' } @@ -163,7 +182,7 @@ fm_pending_reply_txn_owner_write() { # "$tmp"; then + "ticket=$ticket" | fm_nofollow_write "$tmp"; then rm -f "$tmp" || true return 1 fi @@ -175,25 +194,18 @@ fm_pending_reply_txn_owner_write() { # - local state=$1 corr=$2 home_var=$3 watch_var=$4 rec home state_identity home_state_identity + local state=$1 corr=$2 home_var=$3 watch_var=$4 rec home expected_home local lock_home lock_watch rec=$(fm_pending_reply_path "$state" "$corr") home= [ -f "$rec" ] && home=$(fm_pending_reply_get "$rec" parent_home) - state_identity=$(fm_pending_reply_source_identity "$state") || return 1 + expected_home=$(fm_pending_reply_expected_parent_home "$state") || return 1 if [ -n "$home" ]; then - home_state_identity=$(fm_pending_reply_source_identity "$home/state") || home_state_identity= - [ "$home_state_identity" = "$state_identity" ] || home= - fi - if [ -z "$home" ] && [ -n "${FM_HOME:-}" ]; then - home_state_identity=$(fm_pending_reply_source_identity "$FM_HOME/state") || home_state_identity= - [ "$home_state_identity" = "$state_identity" ] && home=$FM_HOME - fi - if [ -z "$home" ] && [ "$(basename "$state_identity")" = state ]; then - home=$(dirname "$state_identity") + home=$(cd "$home" 2>/dev/null && pwd -P) || home= + [ "$home" = "$expected_home" ] || return 1 fi + [ -n "$home" ] || home=$expected_home [ -n "$home" ] || return 1 - home=$(cd "$home" 2>/dev/null && pwd -P) || return 1 lock_home=$(cat "$state/.watch.lock/fm-home" 2>/dev/null || true) lock_watch=$(cat "$state/.watch.lock/watcher-path" 2>/dev/null || true) if [ "$lock_home" != "$home" ] || [ -z "$lock_watch" ]; then @@ -203,12 +215,17 @@ fm_pending_reply_protocol_scope() { # +fm_pending_reply_txn_lock_acquire() { # [deadline-ms] local state=$1 corr=$2 result_var=$3 lock owner pid identity lock_token phase ticket local existing_pid existing_identity existing_token actual attempt=0 generation local incomplete_signature='' incomplete_seen=0 current_signature local winner winner_ticket winner_token live_owner live_choosing max_ticket - local legacy_present protocol_home protocol_watch + local legacy_present protocol_home protocol_watch max_attempts deadline_ms=${4:-} + max_attempts=${FM_PENDING_REPLY_TXN_ATTEMPTS:-200} + case "$max_attempts" in ''|*[!0-9]*) max_attempts=200 ;; esac + [ "$max_attempts" -ge 1 ] || max_attempts=1 + [ "$max_attempts" -le 200 ] || max_attempts=200 + case "$deadline_ms" in ''|*[!0-9]*) deadline_ms=;; esac fm_pending_reply_protocol_scope "$state" "$corr" protocol_home protocol_watch || return 1 fm_watcher_protocol_gate "$state" "$protocol_home" "$protocol_watch" || return 1 lock=$(fm_pending_reply_txn_lock_path "$state" "$corr") @@ -217,7 +234,11 @@ fm_pending_reply_txn_lock_acquire() { # identity=$(fm_pending_reply_pid_identity "$pid") || return 1 lock_token="$pid-$RANDOM-$(fm_pending_reply_now)" owner="$lock/owner-$lock_token" - while [ "$attempt" -lt 200 ]; do + while [ "$attempt" -lt "$max_attempts" ]; do + if ! fm_pending_reply_cleanup_deadline_check "$deadline_ms"; then + rm -f "$owner" 2>/dev/null || true + return 75 + fi if [ -f "$lock" ] && [ ! -d "$lock" ]; then existing_pid=$(fm_pending_reply_txn_lock_value "$lock" pid) existing_identity=$(fm_pending_reply_txn_lock_value "$lock" identity) @@ -229,7 +250,10 @@ fm_pending_reply_txn_lock_acquire() { # rm -f "$lock" 2>/dev/null || true continue fi - sleep 0.05 + fm_pending_reply_cleanup_deadline_sleep "$deadline_ms" || { + rm -f "$owner" 2>/dev/null || true + return 75 + } attempt=$((attempt + 1)) continue fi @@ -262,7 +286,10 @@ fm_pending_reply_txn_lock_acquire() { # continue fi fi - sleep 0.05 + fm_pending_reply_cleanup_deadline_sleep "$deadline_ms" || { + rm -f "$owner" 2>/dev/null || true + return 75 + } attempt=$((attempt + 1)) continue fi @@ -404,7 +431,10 @@ fm_pending_reply_txn_lock_acquire() { # printf -v "$result_var" '%s' "$lock_token" return 0 fi - sleep 0.05 + fm_pending_reply_cleanup_deadline_sleep "$deadline_ms" || { + rm -f "$owner" 2>/dev/null || true + return 75 + } attempt=$((attempt + 1)) done rm -f "$owner" 2>/dev/null || true @@ -481,15 +511,612 @@ fm_pending_reply_get() { # grep "^${key}=" "$rec" 2>/dev/null | tail -1 | cut -d= -f2- || true } +fm_pending_reply_record_validate() { # + local rec=$1 state=$2 wanted_corr=$3 wanted_task=$4 + local record_corr record_task parent_home parent_status parent_abs expected_home state_abs pending_dir expected_status delivered phase + [ -f "$rec" ] && [ ! -L "$rec" ] || return 1 + awk -F= ' + BEGIN { + allowed["schema"]=1; allowed["corr_id"]=1; allowed["task_id"]=1 + allowed["parent_home"]=1; allowed["parent_status"]=1 + allowed["parent_status_scan_signature"]=1; allowed["request_summary"]=1 + allowed["created_epoch"]=1; allowed["delivered_epoch"]=1; allowed["phase"]=1 + allowed["turn_seen_busy"]=1; allowed["request_turn_completed_epoch"]=1 + allowed["recovery_attempted_epoch"]=1; allowed["recovery_sender_pid"]=1 + allowed["recovery_sender_identity"]=1; allowed["recovery_sent_epoch"]=1 + allowed["recovery_delivery_outcome"]=1; allowed["recovery_turn_seen_busy"]=1 + allowed["recovery_turn_completed_epoch"]=1; allowed["escalated_epoch"]=1 + allowed["resolved_epoch"]=1; allowed["resolved_via"]=1; allowed["retired_epoch"]=1 + allowed["retired_via"]=1; allowed["retired_from"]=1 + allowed["retirement_staged_epoch"]=1; allowed["retirement_history_state"]=1 + allowed["retirement_staged_from"]=1; allowed["retirement_source_state"]=1 + allowed["wrong_home_hits"]=1; allowed["wrong_home_sightings"]=1 + allowed["wrong_home_scan_signature"]=1; allowed["grace_secs"]=1 + required["schema"]=1; required["corr_id"]=1; required["task_id"]=1 + required["parent_home"]=1; required["parent_status"]=1 + required["delivered_epoch"]=1; required["phase"]=1 + valid=1 + } + /^[^=]+=/{ + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + next + } + { valid=0 } + END { + for (key in required) if (!(key in seen)) valid=0 + exit !valid + } + ' "$rec" 2>/dev/null || return 1 + [ "$(fm_pending_reply_get "$rec" schema)" = fm-pending-reply.v1 ] || return 1 + printf '%s' "$wanted_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + state_abs=$(cd "$state" 2>/dev/null && pwd -P) || return 1 + record_corr=$(fm_pending_reply_get "$rec" corr_id) + record_task=$(fm_pending_reply_get "$rec" task_id) + [ "$record_corr" = "$wanted_corr" ] || return 1 + [ "$record_task" = "$wanted_task" ] || return 1 + parent_home=$(fm_pending_reply_get "$rec" parent_home) + parent_status=$(fm_pending_reply_get "$rec" parent_status) + case "$parent_home:$parent_status" in /*:/*) ;; *) return 1 ;; esac + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + expected_home=$(fm_pending_reply_expected_parent_home "$state_abs") || return 1 + [ "$parent_abs" = "$expected_home" ] || return 1 + pending_dir=$(fm_pending_reply_dir "$state_abs") + [ -d "$pending_dir" ] && [ ! -L "$pending_dir" ] || return 1 + expected_status="$state_abs/$wanted_task.status" + [ "$parent_status" = "$expected_status" ] || return 1 + [ ! -L "$expected_status" ] || return 1 + if [ -e "$expected_status" ]; then + [ -f "$expected_status" ] || return 1 + fi + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + case "$delivered" in *[!0-9]*) return 1 ;; esac + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|delivery_unknown|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + *) return 1 ;; + esac + return 0 +} + +# JT secondmate outcome routing uses the existing parent-owned pending-reply +# record and its corr= acknowledgement grammar. This small marker only binds a +# secondmate home to that already-owned record; it is not a second parent +# protocol. The marker is intentionally fail-closed when any field is missing, +# malformed, or no longer matches the live parent record. +fm_pending_reply_secondmate_route_path() { # + printf '%s/state/.fm-jt-parent-route' "$1" +} + +fm_pending_reply_secondmate_route_history_path() { # + printf '%s/state/.fm-jt-parent-route-history.%s' "$1" "$2" +} + +fm_pending_reply_parent_corr_valid() { # + [ -z "$1" ] || printf '%s' "$1" | grep -Eq '^[A-Fa-f0-9]{16}$' +} + +fm_pending_reply_secondmate_route_lock_path() { # + printf '%s/state/.fm-jt-parent-route.lock' "$1" +} + +fm_pending_reply_secondmate_route_shape() { # + awk -F= ' + BEGIN { schema=secondmate_id=parent_home=parent_status=corr_id=0; valid=1 } + NF < 2 { valid=0; next } + $1 == "schema" { schema++; next } + $1 == "secondmate_id" { secondmate_id++; next } + $1 == "parent_home" { parent_home++; next } + $1 == "parent_status" { parent_status++; next } + $1 == "corr_id" { corr_id++; next } + { valid=0 } + END { + if (schema != 1 || secondmate_id != 1 || parent_home != 1 || parent_status != 1 || corr_id != 1) valid=0 + exit !valid + } + ' "$1" 2>/dev/null +} + +fm_pending_reply_secondmate_route_write() { # + local secondmate_home=$1 parent_home=$2 parent_state=$3 secondmate_id=$4 corr=$5 + local marker home_marker route_lock tmp parent_abs state_abs expected_home status_path marker_id route_status=0 + local existing_schema existing_id existing_home existing_status existing_corr existing_rec existing_delivered route_mode + local history_marker + FM_PENDING_REPLY_ROUTE_COMMITTED=0 + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 + case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + marker_id=$(cat "$home_marker" 2>/dev/null || true) + [ "$marker_id" = "$secondmate_id" ] || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + [ -d "$parent_state" ] && [ ! -L "$parent_state" ] || return 1 + state_abs=$(cd "$parent_state" 2>/dev/null && pwd -P) || return 1 + expected_home=$(fm_pending_reply_expected_parent_home "$state_abs") || return 1 + [ "$parent_abs" = "$expected_home" ] || return 1 + status_path="$state_abs/$secondmate_id.status" + [ ! -L "$status_path" ] || return 1 + if [ -e "$status_path" ]; then + [ -f "$status_path" ] || return 1 + fi + fm_pending_reply_parent_corr_valid "$corr" || return 1 + [ -n "$corr" ] || return 1 + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if ! fm_lock_acquire_wait "$route_lock"; then + return 1 + fi + tmp=$(mktemp "$secondmate_home/state/.fm-jt-parent-route.XXXXXX") || { + fm_lock_release "$route_lock" || true + return 1 + } + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { + rm -f "$tmp" + fm_lock_release "$route_lock" || true + return 1 + } + { + printf 'schema=fm-jt-parent-route.v1\n' + printf 'secondmate_id=%s\n' "$secondmate_id" + printf 'parent_home=%s\n' "$parent_abs" + printf 'parent_status=%s\n' "$status_path" + printf 'corr_id=%s\n' "$corr" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; fm_lock_release "$route_lock" || true; return 1; } + chmod 600 "$tmp" 2>/dev/null || true + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ -f "$marker" ] && [ ! -L "$marker" ] \ + && [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ]; then + existing_schema=$(fm_pending_reply_get "$marker" schema) + existing_id=$(fm_pending_reply_get "$marker" secondmate_id) + existing_home=$(fm_pending_reply_get "$marker" parent_home) + existing_status=$(fm_pending_reply_get "$marker" parent_status) + existing_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$existing_schema" = fm-jt-parent-route.v1 ] \ + && [ "$existing_id" = "$secondmate_id" ] \ + && [ "$existing_home" = "$parent_abs" ] \ + && [ "$existing_status" = "$status_path" ] \ + && [ "$existing_corr" = "$corr" ]; then + existing_rec=$(fm_pending_reply_active_path "$state_abs" "$existing_corr") + if ! fm_pending_reply_record_validate "$existing_rec" "$state_abs" "$existing_corr" "$secondmate_id"; then + route_status=1 + else + existing_delivered=$(fm_pending_reply_get "$existing_rec" delivered_epoch) + route_mode=2 + [ -n "$existing_delivered" ] && route_mode=4 + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$existing_corr" "$route_mode" \ + || route_status=1 + fi + if [ "$route_status" = 0 ]; then + FM_PENDING_REPLY_ROUTE_COMMITTED=1 + rm -f "$tmp" + fm_lock_release "$route_lock" || return 1 + return 0 + fi + fi + if [ "$existing_schema" != fm-jt-parent-route.v1 ] \ + || [ "$existing_id" != "$secondmate_id" ]; then + route_status=1 + fi + if [ "$route_status" = 0 ]; then + fm_pending_reply_secondmate_route_validate "$secondmate_home" "$existing_corr" \ + || route_status=1 + if [ "$route_status" = 0 ]; then + case "$FM_PENDING_ROUTE_PHASE" in + resolved|retired) ;; + *) route_status=1 ;; + esac + fi + fi + if [ "$route_status" = 0 ] && [ "$existing_corr" != "$corr" ]; then + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$existing_corr") + [ ! -L "$history_marker" ] || route_status=1 + if [ "$route_status" = 0 ] && [ -e "$history_marker" ]; then + [ -f "$history_marker" ] || route_status=1 + [ "$route_status" = 0 ] && cmp -s "$marker" "$history_marker" || route_status=1 + elif [ "$route_status" = 0 ]; then + ln "$marker" "$history_marker" || route_status=1 + fi + fi + else + route_status=1 + fi + fi + if [ "$route_status" = 0 ]; then + if fm_nofollow_rename "$tmp" "$marker"; then + # shellcheck disable=SC2034 # consumed by fm-send.sh after sourcing this library + FM_PENDING_REPLY_ROUTE_COMMITTED=1 + else + route_status=1 + fi + fi + [ "$route_status" = 0 ] || rm -f "$tmp" + fm_lock_release "$route_lock" || route_status=1 + return "$route_status" +} + +fm_pending_reply_secondmate_route_clear_with_mode() { # [ ] + local secondmate_home=$1 corr=$2 allow_undelivered=${3:-0} expected_id=${4:-} expected_home=${5:-} expected_status=${6:-} + local marker route_lock current_corr history_marker status=0 + FM_PENDING_REPLY_ROUTE_LOCK_CONTENDED=0 + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if ! fm_lock_acquire_wait "$route_lock"; then + FM_PENDING_REPLY_ROUTE_LOCK_CONTENDED=1 + return 1 + fi + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ ! -f "$marker" ] || [ -L "$marker" ] \ + || [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ] \ + || ! fm_pending_reply_secondmate_route_shape "$marker"; then + status=1 + else + current_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$current_corr" != "$corr" ]; then + if ! fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2; then + status=1 + else + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$history_marker" ] && [ ! -L "$history_marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + marker=$history_marker + fi + fi + fi + else + marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + fi + if [ "$status" = 0 ] && fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" "$allow_undelivered"; then + [ -z "$expected_id" ] || [ "$FM_PENDING_ROUTE_SECOND_MATE_ID" = "$expected_id" ] || status=1 + [ -z "$expected_home" ] || [ "$FM_PENDING_ROUTE_PARENT_HOME" = "$expected_home" ] || status=1 + [ -z "$expected_status" ] || [ "$FM_PENDING_ROUTE_PARENT_STATUS" = "$expected_status" ] || status=1 + if [ "$status" = 0 ]; then + pending_status=1 + if [ "$allow_undelivered" = 4 ]; then + pending_status=0 + fm_pending_reply_secondmate_route_has_pending_receipt "$secondmate_home" "$corr" \ + || pending_status=$? + fi + case "$pending_status" in + 0) + # Keep one shared route for all receipts bound to this parent + # correlation. The final reported receipt removes it. + ;; + 1) rm -f "$FM_PENDING_ROUTE_MARKER" || status=1 ;; + *) status=1 ;; + esac + fi + elif [ "$status" = 0 ]; then + status=1 + fi + fm_lock_release "$route_lock" || status=1 + return "$status" +} + +fm_pending_reply_secondmate_route_clear() { # + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 0 +} + +fm_pending_reply_secondmate_route_clear_reported() { # [ ] + fm_pending_reply_secondmate_route_clear_with_mode "$1" "$2" 4 "${3:-}" "${4:-}" "${5:-}" + local status=$? + [ "$status" = 1 ] && [ "${FM_PENDING_REPLY_ROUTE_LOCK_CONTENDED:-0}" = 1 ] && return 75 + return "$status" +} + +fm_pending_reply_secondmate_route_clear_undelivered() { # [deadline-ms] + local secondmate_home=$1 corr=$2 deadline_ms=${3:-} marker route_lock current_corr status=0 marker_removed=0 + local wait_secs remaining_ms now_ms + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + route_lock=$(fm_pending_reply_secondmate_route_lock_path "$secondmate_home") + if [ -n "$deadline_ms" ]; then + case "$deadline_ms" in ''|*[!0-9]*) return 1 ;; esac + now_ms=$(fm_pending_reply_cleanup_retry_now_ms) + remaining_ms=$((deadline_ms - now_ms)) + [ "$remaining_ms" -gt 0 ] || return 75 + wait_secs=$((remaining_ms / 1000)) + else + wait_secs=${FM_LOCK_WAIT_SECS:-30} + fi + if ! FM_LOCK_WAIT_SECS="$wait_secs" fm_lock_acquire_wait "$route_lock"; then + [ -n "$deadline_ms" ] && return 75 + return 1 + fi + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + if [ -e "$marker" ] || [ -L "$marker" ]; then + if [ ! -f "$marker" ] || [ -L "$marker" ] \ + || [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" != 5 ] \ + || ! fm_pending_reply_secondmate_route_shape "$marker"; then + fm_lock_release "$route_lock" || true + return 1 + fi + current_corr=$(fm_pending_reply_get "$marker" corr_id) + if [ "$current_corr" != "$corr" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + else + marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$corr") + if [ ! -e "$marker" ] && [ ! -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 0 + fi + if [ ! -f "$marker" ] || [ -L "$marker" ]; then + fm_lock_release "$route_lock" || true + return 1 + fi + fi + if fm_pending_reply_secondmate_route_validate "$secondmate_home" "$corr" 3; then + if rm -f "$FM_PENDING_ROUTE_MARKER"; then + marker_removed=1 + else + status=1 + fi + else + status=1 + fi + fm_lock_release "$route_lock" || [ "$marker_removed" = 1 ] || status=1 + return "$status" +} + +fm_pending_reply_secondmate_route_validate() { # [] [] + local secondmate_home=$1 wanted_corr=${2:-} allow_undelivered=${3:-0} marker line key value schema marker_id secondmate_id current_corr history_marker + local parent_home parent_status corr parent_abs state_abs expected_home parent_status_dir pending_dir expected_status rec active_rec history_rec history_dir + local phase delivered record_home record_home_abs record_task record_corr home_marker + local seen_schema=0 seen_secondmate_id=0 seen_parent_home=0 seen_parent_status=0 seen_corr=0 + FM_PENDING_ROUTE_PARENT_STATUS= + FM_PENDING_ROUTE_PARENT_HOME= + FM_PENDING_ROUTE_CORR= + FM_PENDING_ROUTE_SECOND_MATE_ID= + FM_PENDING_ROUTE_STATE= + FM_PENDING_ROUTE_PHASE= + FM_PENDING_ROUTE_MARKER= + marker=$(fm_pending_reply_secondmate_route_path "$secondmate_home") + if [ -n "$wanted_corr" ]; then + printf '%s' "$wanted_corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + if [ -e "$marker" ] || [ -L "$marker" ]; then + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + [ "$(awk 'END { print NR + 0 }' "$marker" 2>/dev/null || true)" = 5 ] || return 1 + fm_pending_reply_secondmate_route_shape "$marker" || return 1 + if [ "$allow_undelivered" = 3 ] || [ "$allow_undelivered" = 4 ]; then + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" "$allow_undelivered" || return 1 + else + fm_pending_reply_secondmate_route_validate "$secondmate_home" "" 2 || return 1 + fi + current_corr=$FM_PENDING_ROUTE_CORR + if [ "$current_corr" != "$wanted_corr" ]; then + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") + [ -f "$history_marker" ] && [ ! -L "$history_marker" ] || return 1 + marker=$history_marker + fi + else + history_marker=$(fm_pending_reply_secondmate_route_history_path "$secondmate_home" "$wanted_corr") + [ -f "$history_marker" ] && [ ! -L "$history_marker" ] || return 1 + marker=$history_marker + fi + fi + [ -f "$marker" ] && [ ! -L "$marker" ] || return 1 + schema='' secondmate_id='' parent_home='' parent_status='' corr='' + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + *=*) key=${line%%=*}; value=${line#*=} ;; + *) return 1 ;; + esac + case "$key" in + schema) [ "$seen_schema" = 0 ] || return 1; seen_schema=1; schema=$value ;; + secondmate_id) [ "$seen_secondmate_id" = 0 ] || return 1; seen_secondmate_id=1; secondmate_id=$value ;; + parent_home) [ "$seen_parent_home" = 0 ] || return 1; seen_parent_home=1; parent_home=$value ;; + parent_status) [ "$seen_parent_status" = 0 ] || return 1; seen_parent_status=1; parent_status=$value ;; + corr_id) [ "$seen_corr" = 0 ] || return 1; seen_corr=1; corr=$value ;; + *) return 1 ;; + esac + done < "$marker" + [ "$seen_schema" = 1 ] && [ "$seen_secondmate_id" = 1 ] \ + && [ "$seen_parent_home" = 1 ] && [ "$seen_parent_status" = 1 ] \ + && [ "$seen_corr" = 1 ] || return 1 + [ "$schema" = fm-jt-parent-route.v1 ] || return 1 + [ -z "$wanted_corr" ] || [ "$corr" = "$wanted_corr" ] || return 1 + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 + marker_id=$(cat "$home_marker" 2>/dev/null || true) + case "$marker_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + [ "$secondmate_id" = "$marker_id" ] || return 1 + case "$parent_home" in /*) ;; *) return 1 ;; esac + case "$parent_status" in /*) ;; *) return 1 ;; esac + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + parent_status_dir=${parent_status%/*} + [ -n "$parent_status_dir" ] && [ -d "$parent_status_dir" ] && [ ! -L "$parent_status_dir" ] || return 1 + state_abs=$(cd "$parent_status_dir" 2>/dev/null && pwd -P) || return 1 + expected_home=$(fm_pending_reply_expected_parent_home "$state_abs") || return 1 + [ "$parent_abs" = "$expected_home" ] || return 1 + pending_dir=$(fm_pending_reply_dir "$state_abs") + [ -d "$pending_dir" ] && [ ! -L "$pending_dir" ] || return 1 + history_dir=$(fm_pending_reply_history_dir "$state_abs") + if [ -e "$history_dir" ] || [ -L "$history_dir" ]; then + [ -d "$history_dir" ] && [ ! -L "$history_dir" ] || return 1 + fi + expected_status="$state_abs/$secondmate_id.status" + [ "$parent_status" = "$expected_status" ] || return 1 + [ ! -L "$expected_status" ] || return 1 + if [ -e "$expected_status" ]; then + [ -f "$expected_status" ] || return 1 + fi + active_rec=$(fm_pending_reply_active_path "$state_abs" "$corr") + history_rec="$history_dir/$corr" + rec= + if [ -e "$active_rec" ] || [ -L "$active_rec" ]; then + [ -f "$active_rec" ] && [ ! -L "$active_rec" ] || return 1 + rec=$active_rec + elif [ -e "$history_rec" ] || [ -L "$history_rec" ]; then + [ -f "$history_rec" ] && [ ! -L "$history_rec" ] || return 1 + rec=$history_rec + elif [ "$allow_undelivered" != 3 ] && [ "$allow_undelivered" != 4 ]; then + return 1 + fi + if [ -n "$rec" ]; then + fm_pending_reply_record_validate "$rec" "$state_abs" "$corr" "$secondmate_id" || return 1 + record_home=$(fm_pending_reply_get "$rec" parent_home) + [ -d "$record_home" ] && [ ! -L "$record_home" ] || return 1 + record_home_abs=$(cd "$record_home" 2>/dev/null && pwd -P) || return 1 + [ "$record_home_abs" = "$parent_abs" ] || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + delivery_unknown) + [ "$allow_undelivered" = 2 ] \ + || { [ "$allow_undelivered" = 4 ] && [ -z "$wanted_corr" ]; } \ + || return 1 + ;; + *) return 1 ;; + esac + case "$allow_undelivered" in + 1) + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + ;; + 2) + if [ -z "$delivered" ]; then + case "$phase" in awaiting_report|delivery_unknown) ;; *) return 1 ;; esac + fi + ;; + 3) + [ -z "$delivered" ] || return 1 + [ "$phase" = awaiting_report ] || return 1 + ;; + 4) + if [ -n "$wanted_corr" ]; then + [ -n "$delivered" ] || return 1 + fi + ;; + *) + [ -n "$delivered" ] || return 1 + ;; + esac + elif [ "$allow_undelivered" != 3 ] && [ "$allow_undelivered" != 4 ]; then + return 1 + fi + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_PARENT_HOME=$parent_abs + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_PARENT_STATUS=$expected_status + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_CORR=$corr + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_SECOND_MATE_ID=$secondmate_id + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_STATE=$state_abs + FM_PENDING_ROUTE_PHASE=${phase:-} + FM_PENDING_ROUTE_MARKER=$marker + return 0 +} + +fm_pending_reply_secondmate_receipt_validate() { # + local secondmate_home=$1 secondmate_id=$2 parent_home=$3 parent_status=$4 corr=$5 + local home_marker marker_id parent_abs state_abs expected_home parent_status_dir pending_dir expected_status rec active_rec history_rec history_dir + local delivered phase + FM_PENDING_ROUTE_STATE= + [ -d "$secondmate_home" ] && [ ! -L "$secondmate_home" ] || return 1 + [ -d "$secondmate_home/state" ] && [ ! -L "$secondmate_home/state" ] || return 1 + home_marker="$secondmate_home/.fm-secondmate-home" + [ -f "$home_marker" ] && [ ! -L "$home_marker" ] || return 1 + marker_id=$(cat "$home_marker" 2>/dev/null || true) + case "$secondmate_id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + [ "$marker_id" = "$secondmate_id" ] || return 1 + case "$parent_home" in /*) ;; *) return 1 ;; esac + case "$parent_status" in /*) ;; *) return 1 ;; esac + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + [ -d "$parent_home" ] && [ ! -L "$parent_home" ] || return 1 + parent_abs=$(cd "$parent_home" 2>/dev/null && pwd -P) || return 1 + parent_status_dir=${parent_status%/*} + [ -n "$parent_status_dir" ] && [ -d "$parent_status_dir" ] && [ ! -L "$parent_status_dir" ] || return 1 + state_abs=$(cd "$parent_status_dir" 2>/dev/null && pwd -P) || return 1 + expected_home=$(fm_pending_reply_expected_parent_home "$state_abs") || return 1 + [ "$parent_abs" = "$expected_home" ] || return 1 + pending_dir=$(fm_pending_reply_dir "$state_abs") + [ -d "$pending_dir" ] && [ ! -L "$pending_dir" ] || return 1 + history_dir=$(fm_pending_reply_history_dir "$state_abs") + if [ -e "$history_dir" ] || [ -L "$history_dir" ]; then + [ -d "$history_dir" ] && [ ! -L "$history_dir" ] || return 1 + fi + expected_status="$state_abs/$secondmate_id.status" + [ "$parent_status" = "$expected_status" ] || return 1 + [ ! -L "$parent_status" ] || return 1 + if [ -e "$parent_status" ]; then + [ -f "$parent_status" ] || return 1 + fi + active_rec=$(fm_pending_reply_active_path "$state_abs" "$corr") + history_rec="$history_dir/$corr" + if [ -e "$active_rec" ] || [ -L "$active_rec" ]; then + [ -f "$active_rec" ] && [ ! -L "$active_rec" ] || return 1 + rec=$active_rec + elif [ -e "$history_rec" ] || [ -L "$history_rec" ]; then + [ -f "$history_rec" ] && [ ! -L "$history_rec" ] || return 1 + rec=$history_rec + else + return 1 + fi + fm_pending_reply_record_validate "$rec" "$state_abs" "$corr" "$secondmate_id" || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) + [ -n "$delivered" ] || return 1 + phase=$(fm_pending_reply_get "$rec" phase) + case "$phase" in + awaiting_report|recovery_sending|recovery_sent|recovery_failed|recovery_unknown|escalated|resolved|retired) ;; + *) return 1 ;; + esac + # shellcheck disable=SC2034 # consumed by fm-inactive-reconcile.sh + FM_PENDING_ROUTE_STATE=$state_abs + FM_PENDING_ROUTE_PHASE=$phase + return 0 +} + +fm_pending_reply_secondmate_route_has_pending_receipt() { # + local secondmate_home=$1 corr=$2 outcome_dir rec kind rec_corr + outcome_dir="$secondmate_home/state/terminal-outcomes" + [ -d "$outcome_dir" ] && [ ! -L "$outcome_dir" ] || return 1 + for rec in "$outcome_dir"/*.pending; do + if [ ! -e "$rec" ]; then + [ -L "$rec" ] && return 2 + continue + fi + [ -f "$rec" ] && [ ! -L "$rec" ] || return 2 + kind=$(fm_pending_reply_get "$rec" kind) + [ "$kind" = secondmate ] || continue + rec_corr=$(fm_pending_reply_get "$rec" parent_corr) + [ "$rec_corr" = "$corr" ] && return 0 + done + return 1 +} + fm_pending_reply_corr_reusable() { # - local state=$1 corr=$2 task_id=$3 rec phase + local state=$1 corr=$2 task_id=$3 rec phase delivered printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 rec=$(fm_pending_reply_active_path "$state" "$corr") - [ -f "$rec" ] || return 1 - [ "$(fm_pending_reply_get "$rec" task_id)" = "$task_id" ] || return 1 + fm_pending_reply_record_validate "$rec" "$state" "$corr" "$task_id" || return 1 + delivered=$(fm_pending_reply_get "$rec" delivered_epoch) phase=$(fm_pending_reply_get "$rec" phase) case "$phase" in - awaiting_report|recovery_sending|recovery_sent) return 0 ;; + awaiting_report|delivery_unknown) return 0 ;; + recovery_sending|recovery_sent) [ -n "$delivered" ] || return 1; return 0 ;; esac return 1 } @@ -501,14 +1128,15 @@ fm_pending_reply_set() { # dir=$(dirname "$rec") base=$(basename "$rec") tmp="$dir/.${base}.tmp.$$" - : > "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - "${key}="*) continue ;; - esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$rec" - printf '%s=%s\n' "$key" "$value" >> "$tmp" || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + "${key}="*) continue ;; + esac + printf '%s\n' "$line" + done < "$rec" + printf '%s=%s\n' "$key" "$value" + } | fm_nofollow_write "$tmp" || return 1 mv -f "$tmp" "$rec" } @@ -518,18 +1146,19 @@ fm_pending_reply_set_retirement_stage() { # "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - retirement_staged_epoch=*|retirement_history_state=*|retirement_staged_from=*|retirement_source_state=*) continue ;; - esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$rec" - printf '%s\n' \ - "retirement_staged_epoch=$epoch" \ - "retirement_history_state=$history_state" \ - "retirement_staged_from=$source_phase" \ - "retirement_source_state=$source_state" >> "$tmp" || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + retirement_staged_epoch=*|retirement_history_state=*|retirement_staged_from=*|retirement_source_state=*) continue ;; + esac + printf '%s\n' "$line" + done < "$rec" + printf '%s\n' \ + "retirement_staged_epoch=$epoch" \ + "retirement_history_state=$history_state" \ + "retirement_staged_from=$source_phase" \ + "retirement_source_state=$source_state" + } | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true mv -f "$tmp" "$rec" } @@ -540,13 +1169,14 @@ fm_pending_reply_clear_retirement_stage() { # dir=$(dirname "$rec") base=$(basename "$rec") tmp="$dir/.${base}.unstage.$$" - : > "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - retirement_staged_epoch=*|retirement_history_state=*|retirement_staged_from=*) continue ;; - esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$rec" + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + retirement_staged_epoch=*|retirement_history_state=*|retirement_staged_from=*) continue ;; + esac + printf '%s\n' "$line" + done < "$rec" + } | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true mv -f "$tmp" "$rec" } @@ -605,7 +1235,7 @@ fm_pending_reply_create() { # /dev/null && pwd) || parent_home=$1 ;; esac tmp="$dir/.${corr}.tmp.$$" - cat > "$tmp" < "$tmp" || return 1 + printf '%s=%s\n' "$delivery_state" "$value" | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true mv -f "$tmp" "$marker" } @@ -748,17 +1378,248 @@ fm_pending_reply_reconcile_delivery() { # return 1 } +fm_pending_reply_undelivered_cleanup_meta_path() { # + printf '%s.cleanup-meta' "$(fm_pending_reply_active_path "$1" "$2")" +} + +fm_pending_reply_undelivered_cleanup_meta_valid() { # + local meta=$1 corr=$2 secondmate_home=$3 + [ -f "$meta" ] && [ ! -L "$meta" ] || return 1 + awk -F= ' + BEGIN { schema=corr=home=0; valid=1 } + NF != 2 { valid=0; next } + $1 == "schema" { schema++; next } + $1 == "corr_id" { corr++; next } + $1 == "secondmate_home" { home++; next } + { valid=0 } + END { if (schema != 1 || corr != 1 || home != 1) valid=0; exit !valid } + ' "$meta" 2>/dev/null || return 1 + [ "$(fm_pending_reply_get "$meta" schema)" = fm-undelivered-cleanup.v1 ] || return 1 + [ "$(fm_pending_reply_get "$meta" corr_id)" = "$corr" ] || return 1 + [ "$(fm_pending_reply_get "$meta" secondmate_home)" = "$secondmate_home" ] || return 1 +} + +fm_pending_reply_schedule_undelivered_cleanup() { # + local state=$1 corr=$2 secondmate_home=$3 dir meta tmp + printf '%s' "$corr" | grep -Eq '^[A-Fa-f0-9]{16}$' || return 1 + case "$secondmate_home" in + /*) + case "$secondmate_home" in *$'\n'*|*'='*) return 1 ;; esac + ;; + *) return 1 ;; + esac + [ -d "$state" ] && [ ! -L "$state" ] || return 1 + dir=$(fm_pending_reply_dir "$state") + [ -d "$dir" ] && [ ! -L "$dir" ] || return 1 + meta=$(fm_pending_reply_undelivered_cleanup_meta_path "$state" "$corr") + if [ -e "$meta" ] || [ -L "$meta" ]; then + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" + return $? + fi + tmp=$(mktemp "$dir/.cleanup-meta.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + { + printf 'schema=fm-undelivered-cleanup.v1\n' + printf 'corr_id=%s\n' "$corr" + printf 'secondmate_home=%s\n' "$secondmate_home" + } | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + chmod 600 "$tmp" 2>/dev/null || true + if ln "$tmp" "$meta" 2>/dev/null; then + rm -f "$tmp" + return 0 + fi + rm -f "$tmp" + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" +} + +fm_pending_reply_retry_undelivered_cleanup() { # [deadline-ms] + local state=$1 meta=$2 deadline_ms=${3:-} corr secondmate_home token rc=0 lock_rc + corr=$(fm_pending_reply_get "$meta" corr_id) + secondmate_home=$(fm_pending_reply_get "$meta" secondmate_home) + fm_pending_reply_undelivered_cleanup_meta_valid "$meta" "$corr" "$secondmate_home" || return 1 + fm_pending_reply_txn_lock_acquire "$state" "$corr" token "$deadline_ms" || { + lock_rc=$? + [ "$lock_rc" = 75 ] && return 75 + return "$lock_rc" + } + if ! fm_pending_reply_discard_undelivered "$state" "$corr" 1; then + fm_pending_reply_restore_undelivered "$state" "$corr" || true + rc=1 + elif ! fm_pending_reply_secondmate_route_clear_undelivered "$secondmate_home" "$corr" "$deadline_ms"; then + fm_pending_reply_restore_undelivered "$state" "$corr" || true + rc=1 + elif ! fm_pending_reply_finish_undelivered "$state" "$corr"; then + rc=1 + fi + fm_pending_reply_txn_lock_release "$state" "$corr" "$token" || rc=1 + if [ "$rc" = 0 ]; then + rm -f "$meta" || rc=1 + fi + return "$rc" +} + +fm_pending_reply_cleanup_retry_limit() { + local limit=${FM_PENDING_REPLY_CLEANUP_LIMIT:-8} + case "$limit" in ''|*[!0-9]*) limit=8 ;; esac + [ "$limit" -ge 1 ] || limit=1 + [ "$limit" -le 64 ] || limit=64 + printf '%s' "$limit" +} + +fm_pending_reply_cleanup_retry_budget_secs() { + local budget=${FM_PENDING_REPLY_CLEANUP_BUDGET_SECS:-2} + case "$budget" in ''|*[!0-9]*) budget=2 ;; esac + [ "$budget" -ge 1 ] || budget=1 + [ "$budget" -le 30 ] || budget=30 + printf '%s' "$budget" +} + +fm_pending_reply_cleanup_retry_now_ms() { + local stamp seconds + stamp=$(date +%s%N 2>/dev/null || true) + case "$stamp" in + ''|*[!0-9]*) + seconds=$(date +%s 2>/dev/null || printf '0') + printf '%s000' "$seconds" + ;; + *) printf '%s' "${stamp:0:13}" ;; + esac +} + +fm_pending_reply_cleanup_deadline_check() { + local deadline_ms=$1 now_ms + [ -n "$deadline_ms" ] || return 0 + now_ms=$(fm_pending_reply_cleanup_retry_now_ms) + [ "$now_ms" -lt "$deadline_ms" ] +} + +fm_pending_reply_cleanup_deadline_sleep() { + local deadline_ms=$1 now_ms remaining_ms + if [ -z "$deadline_ms" ]; then + sleep 0.05 + return $? + fi + now_ms=$(fm_pending_reply_cleanup_retry_now_ms) + remaining_ms=$((deadline_ms - now_ms)) + [ "$remaining_ms" -ge 50 ] || return 75 + sleep 0.05 +} + +fm_pending_reply_cleanup_retry_batch() { # + local state=$1 dir=$2 cursor_path cursor cleanup_meta base pass started=0 + local processed=0 last='' limit budget start deadline now lock_attempts tmp cleanup_rc + cursor_path="$state/.cleanup-retry.cursor" + limit=$(fm_pending_reply_cleanup_retry_limit) + budget=$(fm_pending_reply_cleanup_retry_budget_secs) + lock_attempts=${FM_PENDING_REPLY_CLEANUP_LOCK_ATTEMPTS:-20} + case "$lock_attempts" in ''|*[!0-9]*) lock_attempts=20 ;; esac + [ "$lock_attempts" -ge 1 ] || lock_attempts=1 + [ "$lock_attempts" -le 200 ] || lock_attempts=200 + cursor=$(cat "$cursor_path" 2>/dev/null || true) + case "$cursor" in + ''|*/*|*[!A-Za-z0-9._-]*) cursor= ;; + *.cleanup-meta) ;; + *) cursor= ;; + esac + if [ -n "$cursor" ] && [ ! -e "$dir/$cursor" ] && [ ! -L "$dir/$cursor" ]; then + cursor= + fi + start=$(fm_pending_reply_cleanup_retry_now_ms) + deadline=$((start + budget * 1000)) + for pass in 1 2; do + if [ "$pass" = 2 ] && [ -z "$cursor" ]; then + break + fi + for cleanup_meta in "$dir"/*.cleanup-meta; do + [ -e "$cleanup_meta" ] || [ -L "$cleanup_meta" ] || continue + base=${cleanup_meta##*/} + if [ "$pass" = 1 ] && [ -n "$cursor" ] && [ "$started" = 0 ]; then + if [ "$base" = "$cursor" ]; then + started=1 + fi + continue + fi + if [ "$pass" = 2 ] && [ "$base" = "$cursor" ]; then + break + fi + now=$(fm_pending_reply_cleanup_retry_now_ms) + [ "$now" -lt "$deadline" ] || break 2 + cleanup_rc=0 + FM_PENDING_REPLY_TXN_ATTEMPTS="$lock_attempts" \ + fm_pending_reply_retry_undelivered_cleanup "$state" "$cleanup_meta" "$deadline" || cleanup_rc=$? + [ "$cleanup_rc" = 0 ] || break 2 + last=$base + processed=$((processed + 1)) + [ "$processed" -lt "$limit" ] || break 2 + done + done + if [ "$processed" -gt 0 ]; then + tmp=$(mktemp "$state/.cleanup-retry.cursor.XXXXXX") || return 1 + [ -f "$tmp" ] && [ ! -L "$tmp" ] || { rm -f "$tmp"; return 1; } + printf '%s\n' "$last" | fm_nofollow_write "$tmp" || { rm -f "$tmp"; return 1; } + mv -f "$tmp" "$cursor_path" || { rm -f "$tmp"; return 1; } + fi + return 0 +} + # Drop an undelivered expectation after a failed send so transport failure does # not masquerade as a missed report later. fm_pending_reply_discard_undelivered() { # - local state=$1 corr=$2 rec delivered marker + local state=$1 corr=$2 keep_backup=${3:-0} rec delivered marker backup + case "$keep_backup" in 0|1) ;; *) return 1 ;; esac rec=$(fm_pending_reply_path "$state" "$corr") - [ -f "$rec" ] || return 0 + backup="${rec}.cleanup" + [ ! -L "$rec" ] && [ ! -L "$backup" ] || return 1 + if [ ! -e "$rec" ]; then + if [ "$keep_backup" = 0 ] && [ -e "$backup" ]; then + rm -f "$backup" || return 1 + fi + return 0 + fi + [ -f "$rec" ] || return 1 + if [ -e "$backup" ]; then + [ -f "$backup" ] || return 1 + cmp -s "$rec" "$backup" || return 1 + rm -f "$backup" || return 1 + fi delivered=$(fm_pending_reply_get "$rec" delivered_epoch) [ -z "$delivered" ] || return 1 marker=$(fm_pending_reply_delivery_confirmation_path "$state" "$corr") rm -f "$marker" 2>/dev/null || true - rm -f "$rec" + if [ "$keep_backup" = 1 ]; then + ln "$rec" "$backup" || return 1 + fi + rm -f "$rec" || return 1 + [ "$keep_backup" = 1 ] || rm -f "$backup" +} + +fm_pending_reply_restore_undelivered() { # + local state=$1 corr=$2 rec backup + rec=$(fm_pending_reply_path "$state" "$corr") + backup="${rec}.cleanup" + [ ! -L "$rec" ] && [ ! -L "$backup" ] || return 1 + if [ -e "$rec" ]; then + [ -f "$rec" ] && [ -f "$backup" ] || return 1 + cmp -s "$rec" "$backup" || return 1 + rm -f "$backup" + return $? + fi + [ -f "$backup" ] || return 1 + mv "$backup" "$rec" +} + +fm_pending_reply_finish_undelivered() { # + local state=$1 corr=$2 rec backup meta + rec=$(fm_pending_reply_path "$state" "$corr") + backup="${rec}.cleanup" + meta=$(fm_pending_reply_undelivered_cleanup_meta_path "$state" "$corr") + [ ! -L "$backup" ] || return 1 + [ ! -L "$meta" ] || return 1 + if [ -e "$backup" ]; then + [ -f "$backup" ] || return 1 + rm -f "$backup" || return 1 + fi + [ ! -e "$meta" ] || rm -f "$meta" } # 0 if a status line is a correlated acknowledgement for . @@ -868,14 +1729,15 @@ fm_pending_reply_prepare_resolved_handoff() { # "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in retirement_source_state=*|retirement_history_state=*) continue ;; esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$history" - printf '%s\n' \ - "retirement_history_state=$history_state" \ - "retirement_source_state=$source_state" >> "$tmp" || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in retirement_source_state=*|retirement_history_state=*) continue ;; esac + printf '%s\n' "$line" + done < "$history" + printf '%s\n' \ + "retirement_history_state=$history_state" \ + "retirement_source_state=$source_state" + } | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true mv -f "$tmp" "$receipt" } @@ -899,14 +1761,15 @@ fm_pending_reply_promote_resolved_record() { # "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in retirement_source_state=*|retirement_history_state=*) continue ;; esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$record" - printf '%s\n' \ - "retirement_history_state=$history_state" \ - "retirement_source_state=$source_state" >> "$tmp" || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in retirement_source_state=*|retirement_history_state=*) continue ;; esac + printf '%s\n' "$line" + done < "$record" + printf '%s\n' \ + "retirement_history_state=$history_state" \ + "retirement_source_state=$source_state" + } | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true mv "$tmp" "$history" || return 1 fi @@ -1326,7 +2189,7 @@ fm_pending_reply_maybe_escalate_locked() { # [ -n "$parent_status" ] || return 1 mkdir -p "$(dirname "$parent_status")" 2>/dev/null || return 1 if ! grep -Fqx "blocked: $payload" "$parent_status" 2>/dev/null; then - printf 'blocked: %s\n' "$payload" >> "$parent_status" 2>/dev/null || return 1 + printf 'blocked: %s\n' "$payload" | fm_nofollow_append "$parent_status" 2>/dev/null || return 1 fi now=$(fm_pending_reply_now) fm_pending_reply_set "$rec" escalated_epoch "$now" || return 1 @@ -1476,15 +2339,16 @@ fm_pending_reply_tick_one() { # [secondmate- # Never scrapes secondmate conversation; uses only parent status, backend busy # state, and optional secondmate-home wrong-home path checks. fm_pending_reply_tick() { # - local state=$1 dir rec corr task_id phase delivered meta backend target label busy sm_home + local state=$1 dir rec corr task_id phase delivered meta cleanup_meta backend target label busy sm_home local observation observation_task found i local -a observation_tasks=() observation_values=() dir=$(fm_pending_reply_dir "$state") [ -d "$dir" ] || return 0 + fm_pending_reply_cleanup_retry_batch "$state" "$dir" || true for rec in "$dir"/*; do [ -f "$rec" ] || continue case "$(basename "$rec")" in - .*) continue ;; + .*|*.cleanup|*.cleanup-meta) continue ;; esac corr=$(fm_pending_reply_get "$rec" corr_id) [ -n "$corr" ] || corr=$(basename "$rec") @@ -1786,19 +2650,20 @@ fm_pending_reply_prepare_forced_retirement() { # "$tmp" || return 1 - while IFS= read -r line || [ -n "$line" ]; do - case "$line" in - phase=*|retired_epoch=*|retired_via=*|retired_from=*) continue ;; - esac - printf '%s\n' "$line" >> "$tmp" || return 1 - done < "$rec" now=$(fm_pending_reply_now) - printf '%s\n' \ - "retired_epoch=$now" \ - "retired_via=forced-teardown" \ - "retired_from=$staged_from" \ - "phase=retired" >> "$tmp" || return 1 + { + while IFS= read -r line || [ -n "$line" ]; do + case "$line" in + phase=*|retired_epoch=*|retired_via=*|retired_from=*) continue ;; + esac + printf '%s\n' "$line" + done < "$rec" + printf '%s\n' \ + "retired_epoch=$now" \ + "retired_via=forced-teardown" \ + "retired_from=$staged_from" \ + "phase=retired" + } | fm_nofollow_write "$tmp" || return 1 chmod 600 "$tmp" 2>/dev/null || true [ ! -e "$staged" ] || return 1 mv "$tmp" "$staged" diff --git a/bin/fm-run-step-bridge.sh b/bin/fm-run-step-bridge.sh new file mode 100755 index 00000000000..4d03b780058 --- /dev/null +++ b/bin/fm-run-step-bridge.sh @@ -0,0 +1,471 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +run_id_from_output() { + awk ' + function normalize(value) { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", value) + gsub(/^"|"$/, "", value) + if (value ~ /^[A-Za-z0-9._:-]+$/) { + parsed=value + } else { + invalid=1 + } + } + /^run:[[:space:]]*$/ { + if (run_seen) invalid=1 + run_seen=1 + in_run=1 + next + } + /^[^[:space:]]/ { in_run=0 } + in_run && /^[[:space:]]+id:[[:space:]]*/ { + id_count++ + value=$0 + sub(/^[[:space:]]+id:[[:space:]]*/, "", value) + normalize(value) + } + END { + if (!run_seen || invalid || id_count != 1 || parsed == "") exit 1 + print parsed + } + ' "$1" +} + +handoff_value() { + local key=$1 + fm_nofollow_read "$FM_RUN_BINDING_HANDOFF" | awk -F= -v wanted="$key" \ + '$1 == wanted { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }' \ + 2>/dev/null +} + +handoff_valid() { + local schema task incarnation state run_id + [ -f "$FM_RUN_BINDING_HANDOFF" ] && [ ! -L "$FM_RUN_BINDING_HANDOFF" ] || return 1 + schema=$(handoff_value schema) || return 1 + task=$(handoff_value task_id) || return 1 + incarnation=$(handoff_value spawn_incarnation) || return 1 + state=$(handoff_value state) || return 1 + [ "$schema" = fm-jt-run-step-handoff.v1 ] || return 1 + [ "$task" = "$FM_RUN_BINDING_TASK" ] || return 1 + [ "$incarnation" = "$FM_RUN_BINDING_INCARNATION" ] || return 1 + case "$state" in pending) ;; bound) + run_id=$(handoff_value run_id) || return 1 + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + ;; + *) return 1 ;; + esac +} + +metadata_generation_valid() { + local meta=$STATE/$FM_RUN_BINDING_TASK.meta incarnation state handoff contents + FM_RUN_BINDING_METADATA_STATE= + contents=$(fm_nofollow_read "$meta") || return 1 + incarnation=$(printf '%s\n' "$contents" | awk -F= '$1 == "spawn_incarnation" { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }') || return 1 + state=$(printf '%s\n' "$contents" | awk -F= '$1 == "run_binding_state" { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }') || return 1 + handoff=$(printf '%s\n' "$contents" | awk -F= '$1 == "run_binding_handoff" { print substr($0, index($0, "=") + 1); n++ } END { exit(n == 1 ? 0 : 1) }') || return 1 + [ "$incarnation" = "$FM_RUN_BINDING_INCARNATION" ] || return 1 + case "$state" in pending|staged|bound) ;; *) return 1 ;; esac + [ "$handoff" = "${FM_RUN_BINDING_HANDOFF##*/}" ] || return 1 + FM_RUN_BINDING_METADATA_STATE=$state +} + +metadata_staged_run_id() { + local meta=$STATE/$FM_RUN_BINDING_TASK.meta result status contents + contents=$(fm_nofollow_read "$meta") || return 75 + result=$(printf '%s\n' "$contents" | awk -F= -v inc="$FM_RUN_BINDING_INCARNATION" ' + BEGIN { valid=1 } + /^[^=]+=/ { + key=$1 + if (key in seen) valid=0 + seen[key]=1 + value=substr($0, index($0, "=") + 1) + if (key == "run_binding_state") { state=value; state_n++ } + if (key == "run_id") { run_id=value; run_n++ } + if (key == "spawn_incarnation") { stored_inc=value; inc_n++ } + next + } + { valid=0 } + END { + if (!valid || state_n != 1 || run_n != 1 || inc_n != 1 \ + || state != "staged" || stored_inc != inc || run_id == "") exit 75 + print run_id + } + ' 2>/dev/null) + status=$? + case "$status" in + 0) + case "$result" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + printf '%s\n' "$result" + ;; + 75) return 75 ;; + *) return 1 ;; + esac +} + +meta_set_run_binding_state() { + local desired_state=$1 run_id=$2 meta tmp status=0 owner acquired=0 + case "$desired_state" in staged|bound) ;; *) return 1 ;; esac + meta="$STATE/$FM_RUN_BINDING_TASK.meta" + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$FM_TASK_LOCK_PATH" "$owner"; then + : + else + fm_lock_acquire_wait "$FM_TASK_LOCK_PATH" || return 1 + acquired=1 + fi + if [ ! -f "$meta" ] || [ -L "$meta" ]; then + status=1 + elif ! metadata_generation_valid; then + status=1 + else + tmp=$(mktemp "$STATE/.${FM_RUN_BINDING_TASK}.meta-run-binding.XXXXXX") || status=1 + if [ "$status" = 0 ]; then + if ! awk -v run_id="$run_id" -v desired_state="$desired_state" ' + /^run_binding_state=/ { print "run_binding_state=" desired_state; print "run_id=" run_id; found=1; next } + /^run_id=/ { next } + { print } + END { exit(found ? 0 : 1) } + ' "$meta" | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$meta"; then + rm -f "$tmp" + status=1 + fi + fi + fi + if [ "$acquired" = 1 ]; then + fm_lock_release "$FM_TASK_LOCK_PATH" || status=1 + fi + return "$status" +} + +meta_stage_run_id() { + meta_set_run_binding_state staged "$1" +} + +meta_bind_run_id() { + meta_set_run_binding_state bound "$1" +} + +publish_run_id() { + local run_id=$1 existing staged_existing status=0 existing_status staged_status owner old_owner acquired=0 \ + evidence existing_active=0 existing_staged=0 + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + old_owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$old_owner" ] && fm_lock_points_to_owner "$FM_TASK_LOCK_PATH" "$old_owner"; then + owner=$old_owner + else + fm_lock_acquire_wait "$FM_TASK_LOCK_PATH" || return 1 + acquired=1 + owner=$(fm_lock_link_owner "$FM_TASK_LOCK_PATH" 2>/dev/null) || status=1 + fi + if [ "$status" = 0 ]; then + FM_TASK_LOCK_OWNER=$owner + export FM_TASK_LOCK_OWNER + handoff_valid || status=1 + metadata_generation_valid || status=1 + fi + if [ "$status" = 0 ]; then + if existing=$(fm_run_step_binding_read "$FM_RUN_BINDING_TASK" "$FM_RUN_BINDING_INCARNATION" 2>/dev/null); then + if [ "$existing" = "$run_id" ]; then + existing_active=1 + else + status=1 + fi + else + existing_status=$? + if [ "$existing_status" = 75 ]; then + evidence=$(fm_run_step_binding_path "$FM_RUN_BINDING_TASK") || status=1 + if [ "$status" = 0 ] && { [ -e "$evidence" ] || [ -L "$evidence" ]; }; then + if staged_existing=$(fm_run_step_binding_staged_read \ + "$FM_RUN_BINDING_TASK" "$FM_RUN_BINDING_INCARNATION" 2>/dev/null); then + if [ "$staged_existing" = "$run_id" ]; then + existing_staged=1 + else + status=1 + fi + else + status=1 + fi + elif [ "$status" = 0 ]; then + if [ "${FM_RUN_BINDING_METADATA_STATE:-}" = bound ]; then + status=1 + elif staged_existing=$(metadata_staged_run_id 2>/dev/null); then + [ "$staged_existing" = "$run_id" ] || status=1 + else + staged_status=$? + [ "${FM_RUN_BINDING_METADATA_STATE:-}" = pending ] && [ "$staged_status" = 75 ] || status=1 + fi + fi + else + status=1 + fi + fi + fi + if [ "$status" = 0 ]; then + if [ "$existing_active" = 1 ]; then + meta_bind_run_id "$run_id" || status=1 + else + if [ "$existing_staged" = 0 ]; then + if ! meta_stage_run_id "$run_id"; then + status=1 + elif ! fm_run_step_binding_publish "$FM_RUN_BINDING_TASK" "$run_id" "$FM_RUN_BINDING_INCARNATION"; then + status=1 + fi + fi + if [ "$status" = 0 ] && ! fm_run_step_binding_activate \ + "$FM_RUN_BINDING_TASK" "$run_id" "$FM_RUN_BINDING_INCARNATION"; then + status=1 + fi + if [ "$status" = 0 ] && ! meta_bind_run_id "$run_id"; then + status=1 + fm_run_step_binding_deactivate \ + "$FM_RUN_BINDING_TASK" "$run_id" "$FM_RUN_BINDING_INCARNATION" || true + fi + fi + fi + if [ "$acquired" = 1 ]; then + fm_lock_release "$FM_TASK_LOCK_PATH" || status=1 + fi + if [ -n "$old_owner" ]; then + FM_TASK_LOCK_OWNER=$old_owner + export FM_TASK_LOCK_OWNER + else + unset FM_TASK_LOCK_OWNER + fi + return "$status" +} + +recover_staged_binding() { + local staged old_owner owner acquired=0 status=0 + old_owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$old_owner" ] && fm_lock_points_to_owner "$FM_TASK_LOCK_PATH" "$old_owner"; then + owner=$old_owner + else + fm_lock_acquire_wait "$FM_TASK_LOCK_PATH" || return 1 + acquired=1 + owner=$(fm_lock_link_owner "$FM_TASK_LOCK_PATH" 2>/dev/null) || status=1 + fi + if [ "$status" = 0 ]; then + FM_TASK_LOCK_OWNER=$owner + export FM_TASK_LOCK_OWNER + if ! metadata_generation_valid; then + status=1 + elif [ "${FM_RUN_BINDING_METADATA_STATE:-}" = staged ]; then + staged=$(metadata_staged_run_id 2>/dev/null) || status=1 + [ "$status" -ne 0 ] || publish_run_id "$staged" || status=1 + fi + fi + if [ "$acquired" = 1 ]; then + fm_lock_release "$FM_TASK_LOCK_PATH" || status=1 + fi + if [ -n "$old_owner" ]; then + FM_TASK_LOCK_OWNER=$old_owner + export FM_TASK_LOCK_OWNER + else + unset FM_TASK_LOCK_OWNER + fi + return "$status" +} + +run_axi_proc_group_id() { + local pid=$1 stat rest + case "$pid" in ''|*[!0-9]*) return 1 ;; esac + [ -r "/proc/$pid/stat" ] || return 1 + stat=$(cat "/proc/$pid/stat" 2>/dev/null) || return 1 + rest=${stat##*) } + set -- $rest + [ "$#" -ge 3 ] || return 1 + printf '%s\n' "$3" +} + +run_axi_child_group_is_current() { + local child=$1 expected_start=$2 actual_start pgid + actual_start=$(fm_pid_start "$child" 2>/dev/null) || return 1 + [ "$actual_start" = "$expected_start" ] || return 1 + pgid=$(run_axi_proc_group_id "$child") || return 1 + [ "$pgid" = "$child" ] +} + +run_axi_group_member_identity() { + local child=$1 entry pid stat rest start + for entry in /proc/[0-9]*; do + pid=${entry#/proc/} + case "$pid" in ''|*[!0-9]*) continue ;; esac + [ "$pid" = "$child" ] && continue + [ -r "$entry/stat" ] || continue + stat=$(cat "$entry/stat" 2>/dev/null) || continue + rest=${stat##*) } + set -- $rest + [ "$#" -ge 20 ] || continue + [ "$3" = "$child" ] || continue + start="proc:${20}" + printf '%s|%s\n' "$pid" "$start" + return 0 + done + return 1 +} + +run_axi_group_member_is_current() { + local child=$1 pid=$2 expected_start=$3 actual_start pgid + actual_start=$(fm_pid_start "$pid" 2>/dev/null) || return 1 + [ "$actual_start" = "$expected_start" ] || return 1 + pgid=$(run_axi_proc_group_id "$pid") || return 1 + [ "$pgid" = "$child" ] +} + +run_axi_stop_group() { + local child=$1 expected_start=$2 member_pid=${3:-} member_start=${4:-} attempts=0 + case "$child" in ''|*[!0-9]*) return 1 ;; esac + [ -n "$expected_start" ] || return 1 + if ! run_axi_child_group_is_current "$child" "$expected_start"; then + if ! kill -0 -- "-$child" 2>/dev/null; then + return 0 + fi + run_axi_group_member_is_current "$child" "$member_pid" "$member_start" || return 1 + fi + kill -TERM -- "-$child" 2>/dev/null || true + while kill -0 -- "-$child" 2>/dev/null && [ "$attempts" -lt 100 ]; do + sleep 0.01 + attempts=$((attempts + 1)) + done + if kill -0 -- "-$child" 2>/dev/null; then + if ! run_axi_child_group_is_current "$child" "$expected_start"; then + run_axi_group_member_is_current "$child" "$member_pid" "$member_start" || return 1 + fi + kill -KILL -- "-$child" 2>/dev/null || true + attempts=0 + while kill -0 -- "-$child" 2>/dev/null && [ "$attempts" -lt 100 ]; do + sleep 0.01 + attempts=$((attempts + 1)) + done + fi + ! kill -0 -- "-$child" 2>/dev/null +} + +run_axi_abort_child() { + local child=$1 expected_start=$2 member_pid=${3:-} member_start=${4:-} actual_start + run_axi_stop_group "$child" "$expected_start" "$member_pid" "$member_start" || true + if actual_start=$(fm_pid_start "$child" 2>/dev/null) \ + && [ "$actual_start" = "$expected_start" ]; then + kill -TERM "$child" 2>/dev/null || true + kill -KILL "$child" 2>/dev/null || true + fi + wait "$child" 2>/dev/null || true +} + +run_axi() { + local child child_start child_status=0 run_id published=0 startup_seen=0 tmpdir output_file + local group_member group_member_pid='' group_member_start='' + local startup_wait_secs startup_deadline total_wait_secs total_deadline now + tmpdir=${FM_RUN_BINDING_TMP:-${TMPDIR:-/tmp}} + output_file=$(mktemp "$tmpdir/.fm-run-step-output.XXXXXX") || return 1 + fm_nofollow_spawn_capture "$output_file" "$FM_RUN_BINDING_REAL" "$@" & + child=$! + case "$child" in ''|*[!0-9]*) rm -f "$output_file"; return 1 ;; esac + child_start=$(fm_pid_start "$child" 2>/dev/null) || { + wait "$child" 2>/dev/null || true + cat "$output_file" + rm -f "$output_file" + return 1 + } + startup_wait_secs=${FM_RUN_BINDING_STARTUP_WAIT_SECS:-30} + case "$startup_wait_secs" in ''|*[!0-9]*|0) startup_wait_secs=30 ;; esac + while [ "${startup_wait_secs#0}" != "$startup_wait_secs" ]; do + startup_wait_secs=${startup_wait_secs#0} + done + [ -n "$startup_wait_secs" ] || startup_wait_secs=30 + case "${#startup_wait_secs}" in + 1|2) ;; + 3) [ "$startup_wait_secs" -le 300 ] || startup_wait_secs=300 ;; + *) startup_wait_secs=300 ;; + esac + startup_deadline=$(( $(date +%s) + startup_wait_secs )) + total_wait_secs=${FM_RUN_BINDING_TOTAL_WAIT_SECS:-3600} + case "$total_wait_secs" in ''|*[!0-9]*|0) total_wait_secs=3600 ;; esac + while [ "${total_wait_secs#0}" != "$total_wait_secs" ]; do + total_wait_secs=${total_wait_secs#0} + done + [ -n "$total_wait_secs" ] || total_wait_secs=3600 + case "${#total_wait_secs}" in + 1|2|3) [ "$total_wait_secs" -le 3600 ] || total_wait_secs=3600 ;; + 4) [ "$total_wait_secs" -le 3600 ] || total_wait_secs=3600 ;; + *) total_wait_secs=3600 ;; + esac + total_deadline=$(( $(date +%s) + total_wait_secs )) + while kill -0 "$child" 2>/dev/null; do + if [ -z "$group_member_pid" ] && group_member=$(run_axi_group_member_identity "$child"); then + IFS='|' read -r group_member_pid group_member_start <<< "$group_member" + fi + if run_id=$(run_id_from_output "$output_file") && [ -n "$run_id" ]; then + startup_seen=1 + fi + now=$(date +%s) + if [ "$now" -ge "$total_deadline" ]; then + run_axi_abort_child "$child" "$child_start" "$group_member_pid" "$group_member_start" + cat "$output_file" + rm -f "$output_file" + return 1 + fi + if [ "$startup_seen" = 0 ] && [ "$now" -ge "$startup_deadline" ]; then + run_axi_abort_child "$child" "$child_start" "$group_member_pid" "$group_member_start" + cat "$output_file" + rm -f "$output_file" + return 1 + fi + sleep 0.05 + done + if ! run_axi_stop_group "$child" "$child_start" "$group_member_pid" "$group_member_start"; then + cat "$output_file" + rm -f "$output_file" + return 1 + fi + wait "$child" || child_status=$? + run_id= + if ! run_id=$(run_id_from_output "$output_file") || [ -z "$run_id" ]; then + cat "$output_file" + rm -f "$output_file" + return 1 + fi + if ! publish_run_id "$run_id"; then + cat "$output_file" + rm -f "$output_file" + return 1 + fi + published=1 + cat "$output_file" + rm -f "$output_file" + [ "$published" = 1 ] || return 1 + return "$child_status" +} + +main() { + local command_name=${1:-} + shift || true + [ "$command_name" = wrap ] || return 2 + [ -n "${FM_RUN_BINDING_ROOT:-}" ] || return 1 + [ -n "${FM_RUN_BINDING_STATE:-}" ] || return 1 + [ -n "${FM_RUN_BINDING_TASK:-}" ] || return 1 + [ -n "${FM_RUN_BINDING_INCARNATION:-}" ] || return 1 + [ -n "${FM_RUN_BINDING_HANDOFF:-}" ] || return 1 + FM_ROOT=$FM_RUN_BINDING_ROOT + FM_HOME=${FM_RUN_BINDING_HOME:-$FM_ROOT} + STATE=$FM_RUN_BINDING_STATE + FM_TASK_LOCK_PATH=$STATE/.spawn-$FM_RUN_BINDING_TASK.lock + FM_SESSION_LOCK_BOOTSTRAP=1 + export FM_ROOT FM_HOME STATE FM_TASK_LOCK_PATH FM_SESSION_LOCK_BOOTSTRAP + . "$FM_ROOT/bin/fm-wake-lib.sh" + . "$FM_ROOT/bin/fm-run-step-lib.sh" + FM_RUN_BINDING_REAL=${1:-} + shift || true + [ -x "$FM_RUN_BINDING_REAL" ] || return 1 + if [ "${1:-}" = axi ] && [ "${2:-}" = run ]; then + recover_staged_binding || return 1 + run_axi "$@" + else + "$FM_RUN_BINDING_REAL" "$@" + fi +} + +main "$@" diff --git a/bin/fm-run-step-lib.sh b/bin/fm-run-step-lib.sh new file mode 100644 index 00000000000..3bbda80c68c --- /dev/null +++ b/bin/fm-run-step-lib.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash + +set -o pipefail + +_FM_RUN_STEP_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=bin/fm-safe-io-lib.sh +. "$_FM_RUN_STEP_LIB_DIR/fm-safe-io-lib.sh" + +fm_run_step_binding_path() { + printf '%s/.run-step-incarnation-%s' "$STATE" "$1" +} + +fm_run_step_binding_archive_path() { + printf '%s/.run-step-incarnation-%s.%s' "$STATE" "$1" "$2" +} + +fm_run_step_binding_lock_release() { + local lock=$1 acquired=$2 + if [ "$acquired" = 1 ]; then + fm_lock_release "$lock" + fi +} + +fm_run_step_binding_read_state() { + local evidence=$1 id=$2 incarnation=$3 expected_state=$4 stored_run status + case "$expected_state" in active|staged) ;; *) return 75 ;; esac + stored_run=$(fm_nofollow_read "$evidence" | awk -F= -v task="$id" -v inc="$incarnation" -v expected_state="$expected_state" ' + BEGIN { + allowed["schema"]=1; allowed["task_id"]=1; allowed["run_id"]=1 + allowed["spawn_incarnation"]=1; allowed["state"]=1 + valid=1 + } + /^[^=]+=/ { + key=$1 + if (!(key in allowed) || (key in seen)) valid=0 + seen[key]=1 + value=substr($0, index($0, "=") + 1) + values[key]=value + if (key == "schema" && value != "fm-jt-run-step-incarnation.v1") valid=0 + if (key == "task_id" && value != task) valid=0 + if (key == "run_id" && value == "") valid=0 + if (key == "spawn_incarnation" && value != inc) valid=0 + if (key == "state" && value != expected_state) valid=0 + next + } + { valid=0 } + END { + if (!("schema" in seen) || !("task_id" in seen) || !("run_id" in seen) \ + || !("spawn_incarnation" in seen) || !("state" in seen)) valid=0 + if (valid) print values["run_id"] + else exit 75 + } + ' 2>/dev/null) + status=$? + [ "$status" = 0 ] || return "$status" + case "$stored_run" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + printf '%s\n' "$stored_run" +} + +fm_run_step_binding_read_variant() { + local id=$1 incarnation=$2 expected_state=$3 evidence lock owner acquired=0 stored_run status + case "$id" in ''|*[!A-Za-z0-9._-]*) return 75 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + evidence=$(fm_run_step_binding_path "$id") || return 1 + lock=${FM_TASK_LOCK_PATH:-$STATE/.spawn-$id.lock} + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$lock" "$owner"; then + : + else + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi + stored_run=$(fm_run_step_binding_read_state "$evidence" "$id" "$incarnation" "$expected_state") + status=$? + fm_run_step_binding_lock_release "$lock" "$acquired" || return 1 + [ "$status" = 0 ] || return "$status" + printf '%s\n' "$stored_run" +} + +fm_run_step_binding_read_held() { + local id=$1 incarnation=$2 expected_state=${3:-active} evidence lock owner acquired=0 stored_run status + case "$id" in ''|*[!A-Za-z0-9._-]*) return 75 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + evidence=$(fm_run_step_binding_path "$id") || return 1 + lock=${FM_TASK_LOCK_PATH:-$STATE/.spawn-$id.lock} + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$lock" "$owner"; then + : + elif [ "${FM_RUN_STEP_READ_ONLY:-0}" = 1 ]; then + stored_run=$(fm_run_step_binding_read_state "$evidence" "$id" "$incarnation" "$expected_state") + status=$? + [ "$status" = 0 ] || return "$status" + FM_RUN_STEP_HELD_LOCK= + FM_RUN_STEP_HELD_LOCK_RELEASE=0 + FM_RUN_STEP_HELD_PREVIOUS_OWNER= + FM_RUN_STEP_HELD_VALUE=$stored_run + return 0 + else + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + owner=${FM_LOCK_OWNER_DIR:-} + fi + stored_run=$(fm_run_step_binding_read_state "$evidence" "$id" "$incarnation" "$expected_state") + status=$? + if [ "$status" -ne 0 ]; then + [ "$acquired" = 1 ] && fm_lock_release "$lock" >/dev/null 2>&1 || true + return "$status" + fi + FM_RUN_STEP_HELD_LOCK=$lock + FM_RUN_STEP_HELD_LOCK_RELEASE=$acquired + FM_RUN_STEP_HELD_PREVIOUS_OWNER=${FM_TASK_LOCK_OWNER:-} + # shellcheck disable=SC2034 # consumed by fm-crew-state.sh after this helper returns + FM_RUN_STEP_HELD_VALUE=$stored_run + if [ "$acquired" = 1 ]; then + FM_TASK_LOCK_OWNER=$owner + export FM_TASK_LOCK_OWNER + fi + return 0 +} + +fm_run_step_binding_release_held() { + local lock=${FM_RUN_STEP_HELD_LOCK:-} release=${FM_RUN_STEP_HELD_LOCK_RELEASE:-0} status=0 + [ -n "$lock" ] || return 0 + if [ "$release" = 1 ]; then + fm_lock_release "$lock" || status=1 + fi + if [ -n "${FM_RUN_STEP_HELD_PREVIOUS_OWNER:-}" ]; then + FM_TASK_LOCK_OWNER=$FM_RUN_STEP_HELD_PREVIOUS_OWNER + export FM_TASK_LOCK_OWNER + else + unset FM_TASK_LOCK_OWNER + fi + unset FM_RUN_STEP_HELD_LOCK FM_RUN_STEP_HELD_LOCK_RELEASE FM_RUN_STEP_HELD_PREVIOUS_OWNER FM_RUN_STEP_HELD_VALUE + return "$status" +} + +fm_run_step_binding_read() { + fm_run_step_binding_read_variant "$1" "$2" active +} + +fm_run_step_binding_staged_read() { + fm_run_step_binding_read_variant "$1" "$2" staged +} + +fm_run_step_binding_metadata_validate() { + local id=$1 run_id=$2 incarnation=$3 meta result + meta="$STATE/$id.meta" + result=$(fm_nofollow_read "$meta" | awk -F= -v run_id="$run_id" -v inc="$incarnation" ' + BEGIN { invalid=0 } + /^[^=]+=/ { + key=$1 + value=substr($0, index($0, "=") + 1) + if (key == "run_binding_state") { state=value; state_n++ } + if (key == "run_id") { stored_run=value; run_n++ } + if (key == "spawn_incarnation") { stored_inc=value; inc_n++ } + next + } + { invalid=1 } + END { + if (state_n == 0 && !invalid) { print "legacy"; exit 0 } + if (!invalid && state_n == 1 && run_n == 1 && inc_n == 1 \ + && state == "bound" && stored_run == run_id && stored_inc == inc) { + print "bound" + exit 0 + } + print "invalid" + } + ' 2>/dev/null) || return 1 + case "$result" in + legacy|bound) return 0 ;; + *) return 75 ;; + esac +} + +fm_run_step_binding_validate() { + local id=$1 run_id=$2 incarnation=$3 stored_run + case "$id" in ''|*[!A-Za-z0-9._-]*) return 75 ;; esac + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 75 ;; esac + stored_run=$(fm_run_step_binding_read "$id" "$incarnation") || return $? + [ "$stored_run" = "$run_id" ] || return 75 + fm_run_step_binding_metadata_validate "$id" "$run_id" "$incarnation" +} + +fm_run_step_binding_publish() { + local id=$1 run_id=$2 incarnation=$3 evidence lock owner acquired=0 tmp status=0 + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + evidence=$(fm_run_step_binding_path "$id") || return 1 + lock=${FM_TASK_LOCK_PATH:-$STATE/.spawn-$id.lock} + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$lock" "$owner"; then + : + else + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi + if [ -e "$evidence" ] || [ -L "$evidence" ]; then + status=1 + fi + if [ "$status" = 0 ]; then + tmp=$(mktemp "$STATE/.run-step-incarnation.$id.XXXXXX") || status=1 + fi + if [ "$status" = 0 ]; then + if ! printf 'schema=fm-jt-run-step-incarnation.v1\ntask_id=%s\nrun_id=%s\nspawn_incarnation=%s\nstate=staged\n' \ + "$id" "$run_id" "$incarnation" | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$evidence" 1; then + rm -f "$tmp" + status=1 + fi + fi + fm_run_step_binding_lock_release "$lock" "$acquired" || status=1 + return "$status" +} + +fm_run_step_binding_activate() { + local id=$1 run_id=$2 incarnation=$3 evidence lock owner acquired=0 tmp status=0 + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + evidence=$(fm_run_step_binding_path "$id") || return 1 + lock=${FM_TASK_LOCK_PATH:-$STATE/.spawn-$id.lock} + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$lock" "$owner"; then + : + else + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi + if [ ! -f "$evidence" ] || [ -L "$evidence" ]; then + status=1 + else + tmp=$(mktemp "$STATE/.run-step-incarnation.$id.XXXXXX") || status=1 + if [ "$status" = 0 ]; then + if ! fm_nofollow_read "$evidence" | awk -F= -v task="$id" -v run_id="$run_id" -v inc="$incarnation" ' + BEGIN { valid=1 } + /^[^=]+=/ { + key=$1 + if (key in seen) valid=0 + seen[key]=1 + value=substr($0, index($0, "=") + 1) + if (key == "schema" && value != "fm-jt-run-step-incarnation.v1") valid=0 + if (key == "task_id" && value != task) valid=0 + if (key == "run_id" && value != run_id) valid=0 + if (key == "spawn_incarnation" && value != inc) valid=0 + if (key == "state") { + if (value != "staged") valid=0 + print "state=active" + next + } + print + next + } + { valid=0 } + END { + if (!("schema" in seen) || !("task_id" in seen) || !("run_id" in seen) \ + || !("spawn_incarnation" in seen) || !("state" in seen)) valid=0 + exit(valid ? 0 : 1) + } + ' | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$evidence"; then + rm -f "$tmp" + status=1 + fi + fi + fi + fm_run_step_binding_lock_release "$lock" "$acquired" || status=1 + return "$status" +} + +fm_run_step_binding_deactivate() { + local id=$1 run_id=$2 incarnation=$3 evidence lock owner acquired=0 tmp status=0 + case "$id" in ''|*[!A-Za-z0-9._-]*) return 1 ;; esac + case "$run_id" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + case "$incarnation" in ''|*[!A-Za-z0-9._:-]*) return 1 ;; esac + evidence=$(fm_run_step_binding_path "$id") || return 1 + lock=${FM_TASK_LOCK_PATH:-$STATE/.spawn-$id.lock} + owner=${FM_TASK_LOCK_OWNER:-} + if [ -n "$owner" ] && fm_lock_points_to_owner "$lock" "$owner"; then + : + else + fm_lock_acquire_wait "$lock" || return 1 + acquired=1 + fi + if [ ! -f "$evidence" ] || [ -L "$evidence" ]; then + status=1 + else + tmp=$(mktemp "$STATE/.run-step-incarnation.$id.XXXXXX") || status=1 + if [ "$status" = 0 ]; then + if ! fm_nofollow_read "$evidence" | awk -F= -v task="$id" -v run_id="$run_id" -v inc="$incarnation" ' + BEGIN { valid=1 } + /^[^=]+=/ { + key=$1 + if (key in seen) valid=0 + seen[key]=1 + value=substr($0, index($0, "=") + 1) + if (key == "schema" && value != "fm-jt-run-step-incarnation.v1") valid=0 + if (key == "task_id" && value != task) valid=0 + if (key == "run_id" && value != run_id) valid=0 + if (key == "spawn_incarnation" && value != inc) valid=0 + if (key == "state") { + if (value != "active") valid=0 + print "state=staged" + next + } + print + next + } + { valid=0 } + END { + if (!("schema" in seen) || !("task_id" in seen) || !("run_id" in seen) \ + || !("spawn_incarnation" in seen) || !("state" in seen)) valid=0 + exit(valid ? 0 : 1) + } + ' | fm_nofollow_write "$tmp" \ + || ! fm_nofollow_rename "$tmp" "$evidence"; then + rm -f "$tmp" + status=1 + fi + fi + fi + fm_run_step_binding_lock_release "$lock" "$acquired" || status=1 + return "$status" +} diff --git a/bin/fm-safe-io-lib.sh b/bin/fm-safe-io-lib.sh new file mode 100644 index 00000000000..8e21e7b5e3d --- /dev/null +++ b/bin/fm-safe-io-lib.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +fm_nofollow_write() { + local path=$1 + command -v perl >/dev/null 2>&1 || return 1 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_TRUNC | $nofollow, 0600) or exit 1; + binmode($fh); + local $/; + my $content = // ""; + print $fh $content or exit 1; + close($fh) or exit 1; + ' "$path" +} + +fm_nofollow_append() { + local path=$1 + command -v perl >/dev/null 2>&1 || return 1 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_APPEND | $nofollow, 0600) or exit 1; + binmode($fh); + local $/; + my $content = // ""; + print $fh $content or exit 1; + close($fh) or exit 1; + ' "$path" +} + +fm_nofollow_read() { + local path=$1 + command -v perl >/dev/null 2>&1 || return 1 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + sysopen(my $fh, $path, O_RDONLY | $nofollow) or exit 1; + binmode($fh); + local $/; + my $content = <$fh> // ""; + print STDOUT $content or exit 1; + close($fh) or exit 1; + ' "$path" +} + +fm_nofollow_rename() { + local source=$1 target=$2 exclusive=${3:-0} + command -v perl >/dev/null 2>&1 || return 1 + case "$exclusive" in 0|1) ;; *) return 1 ;; esac + perl -e ' + use Errno qw(ENOENT); + my ($source, $target, $exclusive) = @ARGV; + lstat($source) && !-l($source) && -f($source) or exit 1; + if (lstat($target)) { + (-l($target) || -d($target) || !-f($target)) and exit 1; + $exclusive and exit 1; + } elsif ($! != ENOENT) { + exit 1; + } + rename($source, $target) or exit 1; + ' "$source" "$target" "$exclusive" +} + +fm_nofollow_chmod() { + local path=$1 mode=$2 + command -v perl >/dev/null 2>&1 || return 1 + perl -e ' + use Fcntl qw(:DEFAULT); + my ($path, $mode) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + $mode =~ /\A[0-7]{3,4}\z/ or exit 1; + sysopen(my $fh, $path, O_RDONLY | $nofollow) or exit 1; + my $fd_dir = -d "/dev/fd" ? "/dev/fd" : "/proc/self/fd"; + chmod(oct($mode), "$fd_dir/" . fileno($fh)) == 1 or exit 1; + close($fh) or exit 1; + ' "$path" "$mode" +} + +fm_nofollow_spawn_capture() { + local path=$1 + shift + [ "$#" -gt 0 ] || return 1 + command -v perl >/dev/null 2>&1 || return 1 + exec perl -e ' + use Fcntl qw(:DEFAULT); + use POSIX (); + my ($path, @command) = @ARGV; + my $nofollow = eval { O_NOFOLLOW() }; + defined($nofollow) or exit 1; + POSIX::setpgid(0, 0) == 0 or exit 125; + sysopen(my $fh, $path, O_WRONLY | O_CREAT | O_TRUNC | $nofollow, 0600) or exit 1; + open(STDOUT, ">&", $fh) or exit 1; + open(STDERR, ">&", $fh) or exit 1; + exec @command; + exit 127; + ' "$path" "$@" +} diff --git a/bin/fm-send.sh b/bin/fm-send.sh index 5e885f9937e..75044b1426b 100755 --- a/bin/fm-send.sh +++ b/bin/fm-send.sh @@ -87,7 +87,50 @@ shift MARK_FROM_FIRSTMATE=0 PENDING_REPLY_CORR= PENDING_REPLY_CREATED=0 +PENDING_ROUTE_COMMITTED=0 TARGET_TASK_ID= +TARGET_HOME= + +clear_new_pending_route() { + [ "$PENDING_ROUTE_COMMITTED" = 1 ] || return 0 + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ + && [ -n "$TARGET_HOME" ]; then + fm_pending_reply_secondmate_route_clear_undelivered "$TARGET_HOME" "$PENDING_REPLY_CORR" + fi +} + +discard_new_pending_reply() { + local route_status=0 discard_status=0 + if [ "$PENDING_REPLY_CREATED" = 1 ] \ + && [ -n "$PENDING_REPLY_CORR" ] \ + && [ -n "$TARGET_HOME" ] \ + && ! fm_pending_reply_schedule_undelivered_cleanup \ + "$STATE" "$PENDING_REPLY_CORR" "$TARGET_HOME"; then + discard_status=1 + fi + if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ] \ + && ! fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" 1; then + discard_status=1 + fi + if [ "$discard_status" = 0 ]; then + if clear_new_pending_route; then + fm_pending_reply_finish_undelivered "$STATE" "$PENDING_REPLY_CORR" || route_status=1 + else + route_status=1 + fm_pending_reply_restore_undelivered "$STATE" "$PENDING_REPLY_CORR" || discard_status=1 + fi + elif [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then + fm_pending_reply_restore_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + fi + if [ "$route_status" = 1 ]; then + echo "error: failed to clear the secondmate pending-reply route; undelivered record cleanup continued" >&2 + fi + if [ "$discard_status" = 1 ]; then + echo "error: failed to discard the undelivered pending-reply record" >&2 + fi + [ "$route_status" = 0 ] && [ "$discard_status" = 0 ] +} + case "$RAW_TARGET" in fm-*) meta="$STATE/${RAW_TARGET#fm-}.meta" @@ -136,10 +179,20 @@ else fm_pending_reply_embed_corr "$MESSAGE" "$PENDING_REPLY_CORR" MESSAGE if [ "$PENDING_REPLY_CREATED" = 1 ] \ && ! fm_pending_reply_prepare_delivery "$STATE" "$PENDING_REPLY_CORR"; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true + discard_new_pending_reply || exit 1 echo "error: failed to durably prepare pending-reply delivery for $TARGET_TASK_ID" >&2 exit 1 fi + TARGET_HOME=$(fm_meta_get "$meta" home) + if fm_pending_reply_secondmate_route_write \ + "$TARGET_HOME" "$FM_HOME" "$STATE" "$TARGET_TASK_ID" "$PENDING_REPLY_CORR"; then + PENDING_ROUTE_COMMITTED=${FM_PENDING_REPLY_ROUTE_COMMITTED:-0} + else + PENDING_ROUTE_COMMITTED=${FM_PENDING_REPLY_ROUTE_COMMITTED:-0} + discard_new_pending_reply || exit 1 + echo "error: failed to bind the secondmate pending-reply route for $TARGET_TASK_ID" >&2 + exit 1 + fi fi # Slash commands open a completion popup in some TUIs (verified on codex); # submitting too fast selects nothing, so give the popup time to settle before @@ -171,9 +224,7 @@ else # Type once, submit, verify. Lenient: only a positively-confirmed swallow # (text still in the composer) is an error; an unreadable pane is assumed sent. if ! verdict=$(fm_backend_send_text_submit "$TARGET_BACKEND" "$T" "$MESSAGE" "$retries" "$sleep_s" "$settle"); then - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not sent to $T ($TARGET_BACKEND send failed)" >&2 exit 1 fi @@ -184,33 +235,25 @@ else sleep "$settle" final_after_pending=1 if ! verdict=$(fm_backend_submit_enter "$TARGET_BACKEND" "$T" 1 "$sleep_s" "$MESSAGE"); then - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: final Enter submission to $T failed" >&2 exit 1 fi fi case "$verdict" in pending) - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not submitted to $T (Enter swallowed; text left in composer)" >&2 exit 1 ;; send-failed) - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: text not sent to $T (tmux send-keys failed)" >&2 exit 1 ;; unknown) if [ "$final_after_pending" = 1 ]; then - if [ "$PENDING_REPLY_CREATED" = 1 ] && [ -n "$PENDING_REPLY_CORR" ]; then - fm_pending_reply_discard_undelivered "$STATE" "$PENDING_REPLY_CORR" || true - fi + discard_new_pending_reply || exit 1 echo "error: final Enter submission to $T could not be confirmed" >&2 exit 1 fi diff --git a/bin/fm-session-start.sh b/bin/fm-session-start.sh index b381c5c1c8d..963a8db0c30 100755 --- a/bin/fm-session-start.sh +++ b/bin/fm-session-start.sh @@ -126,6 +126,24 @@ SUBRULE='----------------------------------------------------------------------- section() { printf '\n%s\n%s\n%s\n' "$RULE" "$1" "$RULE"; } subsection() { printf '\n%s\n%s\n' "$1" "$SUBRULE"; } +session_start_confirm_inactive_rows() { + local wake_output=$1 row _epoch _seq _kind _key _payload confirm_status + while IFS= read -r row || [ -n "$row" ]; do + IFS=$(printf '\t') read -r _epoch _seq _kind _key _payload <<< "$row" + case "$_key" in + inactive-outcome:*) + "$SCRIPT_DIR/fm-inactive-reconcile.sh" caller-output-complete \ + "$_key" "$row" >/dev/null 2>&1 || return 1 + confirm_status=0 + "$SCRIPT_DIR/fm-inactive-reconcile.sh" confirm "$_key" "$row" >/dev/null 2>&1 \ + || confirm_status=$? + [ "$confirm_status" = 0 ] || [ "$confirm_status" = 1 ] \ + || return "$confirm_status" + ;; + esac + done <<< "$wake_output" +} + # print_file_or_absent