From f1a4af426d7199c1781bc91ccd143b8e1f732d10 Mon Sep 17 00:00:00 2001
From: Kun Chen <3233006+kunchenguid@users.noreply.github.com>
Date: Fri, 14 Aug 2026 21:32:23 -0700
Subject: [PATCH 1/3] fix(ci): fail hung Herdr behavior runs in 20 minutes
 (#2413)

A wedged family-run step was occupying the runner until the 75-minute
job cap; bound that step so cleanup and timing artifacts still upload.
---
 .github/workflows/ci.yml        |  9 +++++++--
 docs/fm-test-portable-shards.md | 11 ++++++-----
 tests/fm-test-run.test.sh       | 35 +++++++++++++++++++++++++++++++++
 3 files changed, 48 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 297d70ceeb8..5495ec44947 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -170,8 +170,10 @@ jobs:
   tests-herdr:
     name: Behavior tests (Herdr)
     runs-on: ubuntu-latest
-    # Real Herdr is slower than the portable suite; this is a hang tripwire,
-    # not the expected healthy end of the lane (estimate 15-40 min first cut).
+    # Healthy runs finish around 7 minutes. This job cap is a last-resort hang
+    # tripwire, not the expected end of the lane. The family-run step owns the
+    # tighter bound so a wedged suite fails fast with always() cleanup and
+    # timing artifacts still uploaded (docs/fm-test-portable-shards.md).
     timeout-minutes: 75
     steps:
       - uses: actions/checkout@v6
@@ -252,6 +254,9 @@ jobs:
           mkdir -p "$RUNNER_TEMP/fm-herdr"
           bin/fm-herdr-ci-cleanup.sh snapshot "$RUNNER_TEMP/fm-herdr/sessions-before.json"
       - name: Run real-Herdr family (serial, required)
+        # Comfortably above the ~7 min healthy wall and far below the 75 min
+        # job backstop. A hang must fail this step so cleanup still runs.
+        timeout-minutes: 20
         run: |
           set -eu
           mkdir -p "$RUNNER_TEMP/fm-test"
diff --git a/docs/fm-test-portable-shards.md b/docs/fm-test-portable-shards.md
index 5268627c2a2..5cf681a5019 100644
--- a/docs/fm-test-portable-shards.md
+++ b/docs/fm-test-portable-shards.md
@@ -105,10 +105,11 @@ Portable shards, each portable serial shard, and the Herdr lane upload runner-ge
 
 ## Timeouts
 
-| Job | timeout-minutes | Rationale |
-|---|---:|---|
-| portable parallel 1/2 | 10 | The measured shard sums are about three minutes and the timeout is a hang tripwire. |
-| portable serial 1-4 | 15 | Each balanced shard is about five minutes, leaving roughly 3x hang-tripwire margin. |
-| Herdr | 40 | The real-Herdr lane keeps its dedicated timeout. |
+| Lane | Bound | Rationale |
+|---|---|---|
+| portable parallel 1/2 | job `timeout-minutes: 10` | The measured shard sums are about three minutes and the timeout is a hang tripwire. |
+| portable serial 1-4 | job `timeout-minutes: 15` | Each balanced shard is about five minutes, leaving roughly 3x hang-tripwire margin. |
+| Herdr | family-run step `timeout-minutes: 20`; job `timeout-minutes: 75` backstop | Healthy runs finish around 7 minutes, so the step bound is the hang tripwire (cleanup and timing artifacts still upload) while the job cap stays a last-resort backstop. |
 
 Timeouts are hang tripwires rather than expected healthy durations.
+`.github/workflows/ci.yml` owns the exact numbers.
diff --git a/tests/fm-test-run.test.sh b/tests/fm-test-run.test.sh
index 21bdd69ba5f..8fe26e6f476 100755
--- a/tests/fm-test-run.test.sh
+++ b/tests/fm-test-run.test.sh
@@ -627,6 +627,40 @@ SH
   pass "jobs scheduler runs proven scripts; failure propagates; non-proven refused"
 }
 
+test_herdr_ci_family_run_has_a_step_timeout() {
+  # The required Herdr lane's hang tripwire is the family-run *step* bound, not
+  # the 75-minute job cap. Parse the workflow as YAML so nested `with.name`
+  # artifact keys cannot masquerade as the step contract.
+  command -v ruby >/dev/null 2>&1 \
+    || fail "ruby is required to parse .github/workflows/ci.yml as YAML"
+  local json job_timeout step_timeout
+  json=$(ruby -ryaml -rjson -e '
+doc = YAML.load_file(ARGV[0])
+job = doc.fetch("jobs").fetch("tests-herdr")
+step = job.fetch("steps").find { |s|
+  s.is_a?(Hash) && s["name"] == "Run real-Herdr family (serial, required)"
+}
+raise "missing family-run step" if step.nil?
+raise "family-run step has no timeout-minutes" unless step.key?("timeout-minutes")
+puts JSON.generate(
+  "job_timeout" => job.fetch("timeout-minutes"),
+  "step_timeout" => step.fetch("timeout-minutes")
+)
+' "$ROOT/.github/workflows/ci.yml") \
+    || fail "could not parse tests-herdr timeouts from ci.yml"
+  job_timeout=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["job_timeout"])' <<<"$json") \
+    || fail "could not read job timeout from parsed workflow"
+  step_timeout=$(python3 -c 'import json,sys; print(json.load(sys.stdin)["step_timeout"])' <<<"$json") \
+    || fail "could not read step timeout from parsed workflow"
+  [ "$job_timeout" = 75 ] \
+    || fail "tests-herdr job backstop must stay 75 minutes, got $job_timeout"
+  [ "$step_timeout" = 20 ] \
+    || fail "family-run step timeout must be 20 minutes, got $step_timeout"
+  [ "$step_timeout" -lt "$job_timeout" ] \
+    || fail "family-run step timeout must be below the job backstop"
+  pass "Herdr CI family-run step times out at 20 min under a 75 min job backstop"
+}
+
 test_aggregate_json() {
   local tmp a b
   tmp=$(mktemp -d "${TMPDIR:-/tmp}/fm-test-run-aggjson.XXXXXX")
@@ -685,4 +719,5 @@ test_portable_serial_shards_partition_the_serial_lane
 test_portable_serial_shard_lane_refusals
 test_jobs_requires_proven_isolated
 test_jobs_parallel_scheduler_and_failure_propagation
+test_herdr_ci_family_run_has_a_step_timeout
 test_aggregate_json

From 9fc1ee5d40b71a0ee4ac4f020d07af4714c89c3a Mon Sep 17 00:00:00 2001
From: prajwal-395 <prajwal.mohankumar@gmail.com>
Date: Sat, 15 Aug 2026 13:01:46 -0400
Subject: [PATCH 2/3] feat: Add quota memory extraction for agy harness

Extracts model, quota, and reset time from agy pane footer and stores
it persistently in state/ so dispatch rules can read it.
Fails open on ambiguous reads and resets on staleness.
---
 bin/fm-agy-quota-lib.sh        | 130 +++++++++++++++++++++++++++++++++
 bin/fm-watch.sh                |   5 ++
 tests/fm-agy-quota-lib.test.sh |  80 ++++++++++++++++++++
 3 files changed, 215 insertions(+)
 create mode 100644 bin/fm-agy-quota-lib.sh
 create mode 100755 tests/fm-agy-quota-lib.test.sh

diff --git a/bin/fm-agy-quota-lib.sh b/bin/fm-agy-quota-lib.sh
new file mode 100644
index 00000000000..e172d2953a8
--- /dev/null
+++ b/bin/fm-agy-quota-lib.sh
@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+# fm-agy-quota-lib.sh - Quota memory observation and read helpers.
+# Usage: . bin/fm-agy-quota-lib.sh
+
+# fm_agy_parse_reset_time: converts '4h 24m', '1d 2h', etc to seconds.
+fm_agy_parse_reset_time() {
+  local ts="$1"
+  local total=0
+  local d=0 h=0 m=0 s=0
+
+  case "$ts" in
+    *d*) d="${ts%%d*}"; d="${d##* }"; total=$((total + d * 86400)) ;;
+  esac
+  case "$ts" in
+    *h*) h="${ts%%h*}"; h="${h##* }"; total=$((total + h * 3600)) ;;
+  esac
+  case "$ts" in
+    *m*) m="${ts%%m*}"; m="${m##* }"; total=$((total + m * 60)) ;;
+  esac
+  case "$ts" in
+    *s*) s="${ts%%s*}"; s="${s##* }"; total=$((total + s)) ;;
+  esac
+  echo "$total"
+}
+
+# fm_agy_quota_observe: extract and record quota from agy pane footer.
+# Format: Gemini 3.1 Pro (High) | ctx: 10.5% | quota: 94.7% (4h 24m)
+fm_agy_quota_observe() {  # <text> <state_dir>
+  local text="$1" state_dir="$2"
+  case "$text" in
+    *" | quota: "*) ;;
+    *) return 0 ;;
+  esac
+  
+  local pre="${text%% | quota: *}"
+  local nl='
+'
+  local line_start="${pre##*"$nl"}"
+  local model="${line_start%% | ctx: *}"
+  if [ "$model" = "$line_start" ]; then
+    return 0
+  fi
+  
+  local post="${text#* | quota: }"
+  local line_end="${post%%"$nl"*}"
+  local quota="${line_end%% (*}"
+  local reset_time="${line_end#*(}"
+  reset_time="${reset_time%)}"
+  
+  if [ -z "$model" ] || [ -z "$quota" ] || [ -z "$reset_time" ]; then
+    return 0
+  fi
+
+  local safe_model
+  if command -v md5 >/dev/null 2>&1; then
+    safe_model=$(printf '%s' "$model" | md5 -q)
+  else
+    safe_model=$(printf '%s' "$model" | md5sum | cut -d' ' -f1)
+  fi
+  
+  local now
+  now=$(date +%s)
+  
+  # Only write if it actually changed, but doing this requires reading.
+  # Since this is already conditionally executed, writing directly is fine.
+  printf '%s|%s|%s|%s\n' "$model" "$quota" "$reset_time" "$now" > "$state_dir/.agy-quota-$safe_model"
+}
+
+# fm_agy_quota_read: returns the last known value for a model together with its age.
+fm_agy_quota_read() {  # <model> <state_dir> [<now>]
+  local model="$1" state_dir="$2" now_override="$3"
+  local safe_model
+  if command -v md5 >/dev/null 2>&1; then
+    safe_model=$(printf '%s' "$model" | md5 -q)
+  else
+    safe_model=$(printf '%s' "$model" | md5sum | cut -d' ' -f1)
+  fi
+  
+  local file="$state_dir/.agy-quota-$safe_model"
+  if [ ! -f "$file" ]; then
+    echo "unknown"
+    return 0
+  fi
+  
+  local line
+  line=$(cat "$file" 2>/dev/null || true)
+  if [ -z "$line" ]; then
+    echo "unknown"
+    return 0
+  fi
+  
+  local file_model="${line%%|*}"
+  local rest="${line#*|}"
+  local quota="${rest%%|*}"
+  rest="${rest#*|}"
+  local reset_time_str="${rest%%|*}"
+  local obs_ts="${rest#*|}"
+  
+  if [ "$file_model" != "$model" ] || [ -z "$quota" ] || [ -z "$reset_time_str" ] || [ -z "$obs_ts" ]; then
+    echo "unknown"
+    return 0
+  fi
+  
+  local now
+  if [ -n "$now_override" ]; then
+    now="$now_override"
+  else
+    now=$(date +%s)
+  fi
+  
+  local age=$((now - obs_ts))
+  if [ "$age" -lt 0 ]; then
+    age=0
+  fi
+  
+  local reset_seconds
+  reset_seconds=$(fm_agy_parse_reset_time "$reset_time_str")
+  if [ "$reset_seconds" -eq 0 ]; then
+    echo "unknown"
+    return 0
+  fi
+  
+  if [ "$age" -ge "$reset_seconds" ]; then
+    echo "unknown"
+    return 0
+  fi
+  
+  quota="${quota%%%*}"
+  echo "$quota $age"
+}
diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh
index 3f4a57afd65..cdf93e491c3 100755
--- a/bin/fm-watch.sh
+++ b/bin/fm-watch.sh
@@ -88,6 +88,8 @@ mkdir -p "$STATE"
 . "$SCRIPT_DIR/fm-pending-reply-lib.sh"
 # shellcheck source=bin/fm-busy-lib.sh
 . "$SCRIPT_DIR/fm-busy-lib.sh"
+# shellcheck source=bin/fm-agy-quota-lib.sh
+. "$SCRIPT_DIR/fm-agy-quota-lib.sh"
 
 WATCH_LOCK="$STATE/.watch.lock"
 WATCH_PATH="$SCRIPT_DIR/fm-watch.sh"
@@ -1033,6 +1035,9 @@ EOF
     # content cannot suppress stale detection. Read once per window per poll and
     # reused below so a busy verdict is consistent within one cycle.
     if window_is_busy "$w" "$tail40"; then busy_now=0; else busy_now=1; fi
+    if [ "$h" != "$prev" ]; then
+      fm_agy_quota_observe "$tail40" "$STATE"
+    fi
     if [ "$h" = "$prev" ]; then
       n=$(( $(cat "$cf" 2>/dev/null || echo 0) + 1 ))
       echo "$n" > "$cf"
diff --git a/tests/fm-agy-quota-lib.test.sh b/tests/fm-agy-quota-lib.test.sh
new file mode 100755
index 00000000000..e262c9ea5a9
--- /dev/null
+++ b/tests/fm-agy-quota-lib.test.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+set -u
+
+# shellcheck source=tests/lib.sh
+. "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+# shellcheck source=bin/fm-agy-quota-lib.sh
+. "$ROOT/bin/fm-agy-quota-lib.sh"
+
+TMP_ROOT=$(fm_test_tmproot fm-agy-quota-lib)
+
+assert_eq() {
+  local expected=$1 actual=$2
+  if [ "$expected" != "$actual" ]; then
+    fail "expected '$expected' but got '$actual'"
+  fi
+}
+
+echo "Testing fm_agy_parse_reset_time"
+assert_eq "15840" "$(fm_agy_parse_reset_time '4h 24m')"
+assert_eq "2700" "$(fm_agy_parse_reset_time '45m')"
+assert_eq "93600" "$(fm_agy_parse_reset_time '1d 2h')"
+assert_eq "30" "$(fm_agy_parse_reset_time '30s')"
+pass "fm_agy_parse_reset_time logic works"
+
+echo "Testing observe and read valid"
+state="$TMP_ROOT/state1"
+mkdir -p "$state"
+
+footer="Gemini 3.1 Pro (High) | ctx: 10.5% | quota: 94.7% (4h 24m)"
+fm_agy_quota_observe "$footer" "$state"
+
+now=$(date +%s)
+res=$(fm_agy_quota_read "Gemini 3.1 Pro (High)" "$state" "$now")
+assert_eq "94.7 0" "$res"
+
+# Advance time by 10 seconds
+res=$(fm_agy_quota_read "Gemini 3.1 Pro (High)" "$state" "$((now + 10))")
+assert_eq "94.7 10" "$res"
+pass "observe and read valid handles normal cases"
+
+echo "Testing read older than reset window"
+state="$TMP_ROOT/state2"
+mkdir -p "$state"
+
+footer="Claude Opus 4.6 (Thinking) | ctx: 5% | quota: 14.7% (1h 0m)"
+fm_agy_quota_observe "$footer" "$state"
+
+now=$(date +%s)
+
+# 3600 seconds is the window.
+res=$(fm_agy_quota_read "Claude Opus 4.6 (Thinking)" "$state" "$now")
+assert_eq "14.7 0" "$res"
+
+# 3600 seconds later -> reset
+res=$(fm_agy_quota_read "Claude Opus 4.6 (Thinking)" "$state" "$((now + 3600))")
+assert_eq "unknown" "$res"
+pass "older than window returns unknown"
+
+echo "Testing missing, unparseable, ambiguous"
+state="$TMP_ROOT/state3"
+mkdir -p "$state"
+
+# Missing
+res=$(fm_agy_quota_read "No Such Model" "$state" "$(date +%s)")
+assert_eq "unknown" "$res"
+
+# Unparseable reset time
+footer="Bad Model | ctx: 0% | quota: 50% (forever)"
+fm_agy_quota_observe "$footer" "$state"
+res=$(fm_agy_quota_read "Bad Model" "$state" "$(date +%s)")
+assert_eq "unknown" "$res"
+
+# Ambiguous formatting (missing ctx but has quota, observe will reject it)
+footer="Ambiguous Model | quota: 50% (1h)"
+fm_agy_quota_observe "$footer" "$state"
+res=$(fm_agy_quota_read "Ambiguous Model" "$state" "$(date +%s)")
+assert_eq "unknown" "$res"
+pass "missing unparseable ambiguous fail open"
+
+echo "ALL TESTS PASSED"

From 9cda49d57fc3fd57d358cd05764bbd7d180d683e Mon Sep 17 00:00:00 2001
From: prajwal-395 <prajwal.mohankumar@gmail.com>
Date: Sat, 15 Aug 2026 13:41:28 -0400
Subject: [PATCH 3/3] fix: Support set -u with optional now_override arg in
 fm_agy_quota_read

---
 bin/fm-agy-quota-lib.sh        |  2 +-
 tests/fm-agy-quota-lib.test.sh | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/bin/fm-agy-quota-lib.sh b/bin/fm-agy-quota-lib.sh
index e172d2953a8..c1874beb7d4 100644
--- a/bin/fm-agy-quota-lib.sh
+++ b/bin/fm-agy-quota-lib.sh
@@ -68,7 +68,7 @@ fm_agy_quota_observe() {  # <text> <state_dir>
 
 # fm_agy_quota_read: returns the last known value for a model together with its age.
 fm_agy_quota_read() {  # <model> <state_dir> [<now>]
-  local model="$1" state_dir="$2" now_override="$3"
+  local model="$1" state_dir="$2" now_override="${3:-}"
   local safe_model
   if command -v md5 >/dev/null 2>&1; then
     safe_model=$(printf '%s' "$model" | md5 -q)
diff --git a/tests/fm-agy-quota-lib.test.sh b/tests/fm-agy-quota-lib.test.sh
index e262c9ea5a9..db57408e432 100755
--- a/tests/fm-agy-quota-lib.test.sh
+++ b/tests/fm-agy-quota-lib.test.sh
@@ -77,4 +77,15 @@ res=$(fm_agy_quota_read "Ambiguous Model" "$state" "$(date +%s)")
 assert_eq "unknown" "$res"
 pass "missing unparseable ambiguous fail open"
 
+echo "Testing set -u compliance with 2 arguments"
+state="$TMP_ROOT/state4"
+mkdir -p "$state"
+footer="Model Two Args | ctx: 10% | quota: 50% (1h)"
+fm_agy_quota_observe "$footer" "$state"
+# This call must not crash under set -u
+res=$(fm_agy_quota_read "Model Two Args" "$state")
+# It should successfully read the value (age will be 0 as it was just observed)
+assert_eq "50 0" "$res"
+pass "set -u compliance with 2 arguments works"
+
 echo "ALL TESTS PASSED"