From 75298f49866367eb93f5ad539702e35f595ef4b2 Mon Sep 17 00:00:00 2001 From: Charles Brooker Date: Thu, 8 Oct 2026 21:54:37 -0500 Subject: [PATCH 1/6] fix: bound every test script and lab tree with a per-tree process budget The kernel compares the process limit with the user's whole process count, so one runaway scratch tree (a shim whose basename is itself) filled the table and cost every lane its forks. bin/fm-proc-budget.sh sets the limit to the current count plus a headroom (default 1500) for one command tree only. bin/fm-test-run.sh starts each script under it, bin/fm-live-lab.sh starts every lab process under it, and the crewmate brief scaffolds carry a rule that routes ad-hoc shim, lab, and measurement scripts through it. --- bin/fm-brief.sh | 8 ++ bin/fm-live-lab.sh | 5 +- bin/fm-proc-budget.sh | 124 +++++++++++++++++++++ bin/fm-test-run.sh | 20 ++++ docs/scripts.md | 1 + tests/fm-brief.test.sh | 35 ++++++ tests/fm-proc-budget.test.sh | 206 +++++++++++++++++++++++++++++++++++ tests/fm-test-run.test.sh | 54 +++++++++ 8 files changed, 452 insertions(+), 1 deletion(-) create mode 100755 bin/fm-proc-budget.sh create mode 100755 tests/fm-proc-budget.test.sh diff --git a/bin/fm-brief.sh b/bin/fm-brief.sh index 02ce92f3adf..fe313768d4f 100755 --- a/bin/fm-brief.sh +++ b/bin/fm-brief.sh @@ -596,6 +596,12 @@ IFS= read -r -d '' SHARED_INFRA_RULE <<'EOF' || true EOF SHARED_INFRA_RULE=${SHARED_INFRA_RULE%$'\n'} +# One shared string keeps the ship and scout process-budget rule identical. +# The kernel counts the user's whole process table, so one runaway scratch tree +# starves every lane; bin/fm-proc-budget.sh owns the budget, this is the one +# line that reaches ad-hoc scripts no runner wraps. +SHARED_PROC_BUDGET_RULE="9. Run any shim, lab, or measurement script through \`$FM_ROOT/bin/fm-proc-budget.sh -- \`: a runaway process tree then stops at its own limit instead of starving every lane of fork." + if [ -n "$BASE_BRANCH" ]; then SETUP_BASE="You are in a disposable git worktree of $REPO, at a detached HEAD on a clean copy of its base branch. Base branch: $BASE_BRANCH" @@ -661,6 +667,7 @@ $CREWMATE_PAUSE_INSTRUCTIONS Firstmate's reply normally writes that closing line at answer time; when a blocker or wait clears WITHOUT a firstmate reply, append \`resolved [at=]: {how it cleared}\` yourself (same \`[key=]\` if you opened it with one) as you resume. $SHARED_INFRA_RULE $PRIVATE_SERVICE_RULE +$SHARED_PROC_BUDGET_RULE $WAIT_BLOCK$INBOX_SECTION @@ -742,6 +749,7 @@ $ASK_USER_BLOCK Firstmate's reply normally writes that closing line at answer time; when a blocker or wait clears WITHOUT a firstmate reply, append \`resolved [at=]: {how it cleared}\` yourself (same \`[key=]\` if you opened it with one) as you resume. $SHARED_INFRA_RULE $PRIVATE_SERVICE_RULE +$SHARED_PROC_BUDGET_RULE $WAIT_BLOCK$INBOX_SECTION diff --git a/bin/fm-live-lab.sh b/bin/fm-live-lab.sh index 4b6176a2acf..feb5a3ffbfe 100755 --- a/bin/fm-live-lab.sh +++ b/bin/fm-live-lab.sh @@ -172,11 +172,14 @@ lab_env_base() { [ -z "${CLAUDE_DIR:-}" ] || printf '%s\n' "CLAUDE_CONFIG_DIR=$CLAUDE_DIR" } +# Every lab process starts under bin/fm-proc-budget.sh, so a runaway process tree +# in the lab (the private tmux server carries the budget to every pane) stops at +# its own limit instead of starving the user's other sessions of fork. lab_run() { # [NAME=VALUE...] : run in the lab's clean environment local -a base=() local line while IFS= read -r line; do base+=("$line"); done < <(lab_env_base) - env -i "${base[@]}" "$@" + "$SCRIPT_DIR/fm-proc-budget.sh" -- env -i "${base[@]}" "$@" } # window_id : the tmux id of the lab window with exactly this name, or diff --git a/bin/fm-proc-budget.sh b/bin/fm-proc-budget.sh new file mode 100755 index 00000000000..815b4ead6dc --- /dev/null +++ b/bin/fm-proc-budget.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# fm-proc-budget.sh - run a command tree under a per-tree process budget. +# +# Usage: +# fm-proc-budget.sh [] -- run with a budget +# fm-proc-budget.sh --count print the user's process count +# fm-proc-budget.sh --check exit 0 when a budget can be set here +# +# The kernel compares RLIMIT_NPROC with the user's WHOLE process count, not with +# one process tree. A runaway tree therefore starves every other process of the +# same user until the cap is full, and a bash 3.2 script exits 128 at its first +# failed fork. The only handle is to stop the runaway early: this wrapper reads +# the current count and sets the limit (soft and hard) to count + , then +# execs the command, so only the tree below it carries the lowered limit and the +# rest of the user keeps its headroom. is a positive decimal integer; +# omit it for FM_PROC_BUDGET_EXTRA, or 1500 when that is unset. +# +# The budget only tightens. A limit the command already inherits that is lower +# than count + stays as it is, and the hard limit is lowered with the +# soft one so the tree cannot raise it again; a nested wrapper therefore never +# widens an outer budget. +# +# Only commands started through the wrapper are protected. Route every shim, +# lab, and measurement script through it; bin/fm-test-run.sh and +# bin/fm-live-lab.sh do so for the trees they start. +# +# FM_PROC_BUDGET=off runs the command with no budget, for a host where the +# budget turns out to starve a legitimate tree. +# +# The budget is headroom over the count at the moment the command starts, so a +# tree that outlives a large rise in the rest of the user's processes can meet +# the limit without having grown. +# +# Exit status: the command's own, 2 for a malformed call, and 125 when no budget +# can be set (an unreadable process count, or a limit the host refuses). The +# command never runs unbudgeted behind the caller's back in those two cases. +set -u + +fm_proc_budget_error() { + echo "fm-proc-budget: $*" >&2 +} + +fm_proc_budget_usage() { + fm_proc_budget_error "usage: fm-proc-budget.sh [] -- | --count | --check" +} + +# The kernel counts what RLIMIT_NPROC is compared with: processes on Darwin, +# tasks (threads included) on Linux. +fm_proc_budget_count() { + local uid count + uid=$(id -u) || return 1 + set -o pipefail + case "$(uname -s)" in + Linux) count=$(ps -L -U "$uid" -o lwp= | wc -l) || return 1 ;; + *) count=$(ps -U "$uid" -o pid= | wc -l) || return 1 ;; + esac + count=$(printf '%s' "$count" | tr -d '[:space:]') + case "$count" in + ''|*[!0-9]*|0) return 1 ;; + esac + printf '%s\n' "$count" +} + +# fm_proc_budget_apply : lower this process's limit to count + +# unless the inherited limit is already lower. Prints nothing on success. +fm_proc_budget_apply() { + local extra=$1 count current limit + count=$(fm_proc_budget_count) || { + fm_proc_budget_error "cannot read the process count, so no budget can be set" + return 125 + } + limit=$((count + extra)) + current=$(ulimit -u) || { + fm_proc_budget_error "cannot read the current process limit" + return 125 + } + case "$current" in + ''|*[!0-9]*) ;; # unlimited: the budget is the only bound + *) [ "$limit" -lt "$current" ] || return 0 ;; + esac + ulimit -u "$limit" 2>/dev/null || { + fm_proc_budget_error "this host refused a process limit of $limit (count $count + $extra)" + return 125 + } +} + +case "${1:-}" in + --count) + fm_proc_budget_count || { + fm_proc_budget_error "cannot read the process count" + exit 125 + } + exit 0 + ;; + --check) + # The limit is applied in a subshell, so this process keeps its own. + ( fm_proc_budget_apply 1500 ) || exit 1 + echo "fm-proc-budget: a process budget can be set on this host" + exit 0 + ;; +esac + +extra=${FM_PROC_BUDGET_EXTRA:-1500} +if [ "${1:-}" != -- ] && [ "$#" -gt 0 ]; then + extra=$1 + shift +fi +if [ "${1:-}" != -- ]; then + fm_proc_budget_usage + exit 2 +fi +shift +[ "$#" -gt 0 ] || { fm_proc_budget_usage; exit 2; } +case "$extra" in + ''|*[!0-9]*|0|0[0-9]*) + fm_proc_budget_error "extra must be a positive decimal integer, got '$extra'" + exit 2 + ;; +esac + +if [ "${FM_PROC_BUDGET:-}" != off ]; then + fm_proc_budget_apply "$extra" || exit 125 +fi +exec "$@" diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 82594584617..90d3efc1522 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -134,6 +134,13 @@ # The marker must be a nonnegative decimal integer or execution exits 125; # 0 denotes degraded work and imposes no budget. # Without python3 or the pool tool beside it, scripts run directly. +# +# Every executed script also runs under a per-tree process budget +# (bin/fm-proc-budget.sh owns it): a script that forks without bound stops at its +# own limit instead of filling the user's process table for every other lane. +# The budget is taken when the script starts, after any pass wait. FM_PROC_BUDGET=off +# disables it, FM_PROC_BUDGET_EXTRA sizes it, and a host that cannot set one runs +# scripts without it and says so on stderr. # With a usable pool, --jobs above its size still starts that many workers, # but only pool-size scripts run at once. # @@ -849,6 +856,7 @@ tests/fm-pr-check-security.test.sh 300675 tests/fm-pr-reviewers.test.sh 273 tests/fm-pr-state-live-e2e.test.sh 47 tests/fm-pr-state.test.sh 531 +tests/fm-proc-budget.test.sh 8000 tests/fm-procevent-quota.test.sh 2459 tests/fm-procevent-when.test.sh 25674 tests/fm-procevent.test.sh 370820 @@ -2467,6 +2475,13 @@ elif ! command -v python3 >/dev/null 2>&1; then CPU_PASS_ACTIVE=0 log "running without CPU passes: python3 not found" fi +PROC_BUDGET_ACTIVE=1 +if [ ! -x "$ROOT/bin/fm-proc-budget.sh" ] || [ "${FM_PROC_BUDGET:-}" = off ]; then + PROC_BUDGET_ACTIVE=0 +elif ! proc_budget_note=$("$ROOT/bin/fm-proc-budget.sh" --check 2>&1); then + PROC_BUDGET_ACTIVE=0 + log "running without process budgets: $proc_budget_note" +fi if [ "$JOBS" -gt 1 ] || [ "$MODE" = changed ] || [ "$MODE" = scripts ]; then SELECTION_DESC="${SELECTION_DESC};jobs=$JOBS" fi @@ -2647,6 +2662,11 @@ run_script_bounded() { #