Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 2 additions & 27 deletions hermes_cli/kanban_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -6543,16 +6543,6 @@ def schedule_task(
# for operators who want a tighter/looser probe cadence.
DEFAULT_RATE_LIMIT_COOLDOWN_SECONDS = 300 # 5 minutes

# Within this window a GitHub PR URL in a comment blocks re-spawn.
_RESPAWN_GUARD_PR_WINDOW = 86400 # 24 hours

# Pattern matching a GitHub PR URL in task comments.
_RESPAWN_GUARD_PR_URL_RE = re.compile(
r"https?://github\.com/[^/\s]+/[^/\s]+/pull/\d+",
re.IGNORECASE,
)


@dataclass
class DispatchResult:
"""Outcome of a single ``dispatch`` pass."""
Expand Down Expand Up @@ -6597,9 +6587,8 @@ class DispatchResult:
respawn_guarded: list[tuple[str, str]] = field(default_factory=list)
"""Tasks skipped by the respawn guard, as ``(task_id, reason)`` pairs.

Reasons: ``"blocker_auth"`` (quota/auth error — also auto-blocked),
``"recent_success"`` (completed run within guard window),
``"active_pr"`` (GitHub PR URL in a recent comment)."""
Reasons: ``"blocker_auth"`` (quota/auth error) and
``"recent_success"`` (completed run within guard window)."""
rate_limited: list[str] = field(default_factory=list)
"""Task ids whose workers bailed on a provider rate-limit / quota wall
(EX_TEMPFAIL sentinel exit) and were released back to ``ready`` WITHOUT
Expand Down Expand Up @@ -7814,11 +7803,6 @@ def check_respawn_guard(conn: sqlite3.Connection, task_id: str) -> Optional[str]
explicit re-queue event (status change, promote, unblock, reclaim)
arrives AFTER that completion — that's a deliberate re-run request.

``"active_pr"``
A GitHub PR URL appears in a recent task comment (within
``_RESPAWN_GUARD_PR_WINDOW`` seconds). A prior worker already
opened a PR; re-spawning risks a duplicate PR on the same task.

Stale / dead claim locks are NOT a guard reason — they are handled
by ``release_stale_claims`` and ``detect_crashed_workers`` which
reset the task to ``ready`` only after verifying the lock is
Expand Down Expand Up @@ -7899,15 +7883,6 @@ def check_respawn_guard(conn: sqlite3.Connection, task_id: str) -> Optional[str]
if not requeued_after:
return "recent_success"

# 4. GitHub PR URL in a recent comment — prior worker already opened a PR.
pr_cutoff = now - _RESPAWN_GUARD_PR_WINDOW
for c in conn.execute(
"SELECT body FROM task_comments WHERE task_id = ? AND created_at >= ?",
(task_id, pr_cutoff),
).fetchall():
if c["body"] and _RESPAWN_GUARD_PR_URL_RE.search(c["body"]):
return "active_pr"

return None


Expand Down
72 changes: 48 additions & 24 deletions tests/hermes_cli/test_kanban_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -1917,28 +1917,13 @@ def test_respawn_guard_stale_success_not_guarded(kanban_home):
assert reason is None


def test_respawn_guard_active_pr_in_comment(kanban_home):
"""A GitHub PR URL in a recent comment triggers active_pr."""
def test_respawn_guard_pr_comment_is_delivery_metadata(kanban_home):
"""A PR URL alone does not override the card's runnable lifecycle state."""
with kb.connect() as conn:
t = kb.create_task(conn, title="has-pr", assignee="alice")
kb.add_comment(
conn, t, "worker",
"PR created: https://github.com/totemx-AI/subsidysmart/pull/42",
)
reason = kb.check_respawn_guard(conn, t)
assert reason == "active_pr"


def test_respawn_guard_old_pr_comment_not_guarded(kanban_home):
"""A GitHub PR URL in a comment older than the PR window does not block."""
with kb.connect() as conn:
t = kb.create_task(conn, title="old-pr", assignee="alice")
old_ts = int(time.time()) - kb._RESPAWN_GUARD_PR_WINDOW - 60
conn.execute(
"INSERT INTO task_comments (task_id, author, body, created_at) "
"VALUES (?, 'worker', "
"'PR: https://github.com/totemx-AI/subsidysmart/pull/10', ?)",
(t, old_ts),
"PR created: https://github.com/example/project/pull/42",
)
reason = kb.check_respawn_guard(conn, t)
assert reason is None
Expand Down Expand Up @@ -2015,10 +2000,10 @@ def fake_spawn(task, workspace):
assert kb.get_task(conn, t).status == "ready" # not blocked, just skipped


def test_dispatch_respawn_guard_skips_active_pr(
def test_dispatch_pr_comment_does_not_change_ready_spawnability(
kanban_home, all_assignees_spawnable
):
"""dispatch_once skips (but does not block) a task with an active PR comment."""
"""A ready card remains runnable after a worker records its PR URL."""
spawned_ids = []

def fake_spawn(task, workspace):
Expand All @@ -2028,15 +2013,54 @@ def fake_spawn(task, workspace):
t = kb.create_task(conn, title="has-pr", assignee="alice")
kb.add_comment(
conn, t, "worker",
"Opened https://github.com/totemx-AI/subsidysmart/pull/99",
"Opened https://github.com/example/project/pull/99",
)
res = kb.dispatch_once(conn, spawn_fn=fake_spawn)

assert (t, "active_pr") in res.respawn_guarded
assert t not in spawned_ids
assert t in spawned_ids
assert not res.respawn_guarded
assert t not in res.auto_blocked
with kb.connect() as conn:
assert kb.get_task(conn, t).status == "ready"
task = kb.get_task(conn, t)
assert task is not None
assert task.status == "running"


def test_pr_delivery_metadata_defers_to_block_and_explicit_continuation(
kanban_home, all_assignees_spawnable
):
"""Blocked cards stay parked; unblocking resumes the same card and workspace."""
spawned = []

def fake_spawn(task, workspace):
spawned.append((task.id, workspace))

with kb.connect() as conn:
t = kb.create_task(conn, title="review-fix", assignee="alice")
kb.add_comment(
conn, t, "worker",
"Draft PR: https://github.com/example/project/pull/100",
)
task = kb.get_task(conn, t)
assert task is not None
workspace = kb.resolve_workspace(task)
assert kb.block_task(conn, t, reason="review-required")

parked = kb.dispatch_once(conn, spawn_fn=fake_spawn)
assert not parked.spawned
assert not spawned
task = kb.get_task(conn, t)
assert task is not None
assert task.status == "blocked"

assert kb.unblock_task(conn, t)
resumed = kb.dispatch_once(conn, spawn_fn=fake_spawn)

assert [task_id for task_id, _, _ in resumed.spawned] == [t]
assert spawned == [(t, str(workspace))]
task = kb.get_task(conn, t)
assert task is not None
assert task.status == "running"


def test_dispatch_respawn_guard_dry_run_no_auto_block(
Expand Down
6 changes: 4 additions & 2 deletions website/docs/user-guide/features/kanban.md
Original file line number Diff line number Diff line change
Expand Up @@ -744,7 +744,9 @@ hermes kanban create "nightly backup audit" \

### Respawn guard

The dispatcher refuses to re-spawn a ready task when it hit a quota/auth/429 error on the previous run (`blocker_auth`), or completed a run successfully within the guard window (`recent_success`), or a recent task comment links to a GitHub PR (`active_pr`). This prevents repeat worker storms on the same bug or task while a human catches up. See the `respawn_guarded` row in the [event reference](#event-reference).
The dispatcher refuses to re-spawn a ready task when it hit a quota/auth/429 error on the previous run (`blocker_auth`) or completed a run successfully within the guard window (`recent_success`). This prevents repeat worker storms while an external limit clears or a completed result awaits an explicit rerun. See the `respawn_guarded` row in the [event reference](#event-reference).

A pull-request URL in a task comment is delivery and audit metadata, not a dispatch guard. Lifecycle state remains authoritative: a task parked in `blocked` (for example, while awaiting review) is not dispatchable, and explicitly unblocking that same task returns it to the normal dependency and claim checks. Existing claim locks, worker PIDs, and heartbeats still prevent a live worker from being started twice.

### Drag-to-delete and bulk delete (dashboard)

Expand Down Expand Up @@ -965,7 +967,7 @@ Every transition appends a row to `task_events`. Each row carries an optional `r
| `crashed` | `{pid, claimer}` | Worker PID no longer alive but TTL hadn't expired yet. |
| `timed_out` | `{pid, elapsed_seconds, limit_seconds, sigkill}` | `max_runtime_seconds` exceeded; dispatcher SIGTERM'd (then SIGKILL'd after 5 s grace) and re-queued. |
| `stale` | `{elapsed_seconds, last_heartbeat_at, heartbeat_age_seconds, timeout_seconds, pid, terminated}` | Task ran longer than `kanban.dispatch_stale_timeout_seconds` (default 4 h) AND no `kanban_heartbeat` arrived in the last hour. Dispatcher SIGTERM'd the host-local worker (if any), reset the task to `ready` for re-dispatch. Does NOT tick the failure counter (stale is dispatcher-side absence detection, not a worker fault). Workers running long operations should call `kanban_heartbeat` at least once an hour to avoid this. |
| `respawn_guarded` | `{reason}` | Dispatcher refused to re-spawn this ready task this tick. Reasons: `blocker_auth` (last failure was a quota/auth/429 error — wait for the rate window to reset), `recent_success` (a completed run happened in the last hour — wait for review before re-running), `active_pr` (a GitHub PR URL appears in a recent comment — a prior worker already opened a PR). The task stays in `ready`; the next tick gets another chance to spawn. If the underlying condition persists, the normal `consecutive_failures` circuit breaker will auto-block via `gave_up` after `failure_limit` failures. |
| `respawn_guarded` | `{reason}` | Dispatcher refused to re-spawn this ready task this tick. Reasons: `blocker_auth` (last failure was a quota/auth/429 error — wait for the rate window to reset) and `recent_success` (a completed run happened in the last hour — use an explicit lifecycle transition to request another run). Pull-request comments are audit metadata and do not emit this event. The task stays in `ready`; the next tick gets another chance to spawn. If the underlying condition persists, the normal `consecutive_failures` circuit breaker will auto-block via `gave_up` after `failure_limit` failures. |
| `spawn_failed` | `{error, failures}` | One spawn attempt failed (missing PATH, workspace unmountable, …). Counter increments; task returns to `ready` for retry. |
| `protocol_violation` | `{pid, claimer, exit_code, protocol_violation}` | Worker exited successfully while the task was still `running`, usually because it answered without calling `kanban_complete` or `kanban_block`. Emitted on every violation (the payload's `protocol_violation: true` marker is copied into the run metadata and feeds the violation-only retry budget). Below the budget — up to `_PROTOCOL_VIOLATION_FAILURE_LIMIT` (default 3) *consecutive* violations, per-task `max_retries` overriding — the task simply returns to `ready` for another attempt; when the streak reaches the bound the dispatcher also emits `gave_up` and auto-blocks. |
| `gave_up` | `{failures, effective_limit, limit_source, error}` | Circuit breaker fired after N consecutive non-successful attempts. Task auto-blocks with the last error. The effective limit resolves as task `max_retries`, then dispatcher `failure_limit` / `kanban.failure_limit`, then the built-in default. |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -753,7 +753,7 @@ hermes kanban runs t_abcd
| `crashed` | `{pid, claimer}` | Worker PID 不再存活但 TTL 尚未过期。 |
| `timed_out` | `{pid, elapsed_seconds, limit_seconds, sigkill}` | 超过 `max_runtime_seconds`;调度器发送 SIGTERM(5 秒宽限后发送 SIGKILL)并重新排队。 |
| `stale` | `{elapsed_seconds, last_heartbeat_at, heartbeat_age_seconds, timeout_seconds, pid, terminated}` | 任务运行时间超过 `kanban.dispatch_stale_timeout_seconds`(默认 4 小时)**且**最近一小时内没有 `kanban_heartbeat`。调度器向本地 worker(如有)发送 SIGTERM,将任务重置为 `ready` 重新调度。**不**增加失败计数器(stale 是调度器端的缺席检测,不是 worker 故障)。运行长时间操作的 Worker 应至少每小时调用一次 `kanban_heartbeat` 以避免此情况。 |
| `respawn_guarded` | `{reason}` | 调度器拒绝在本 tick 重新启动此就绪任务。原因:`blocker_auth`(上次失败是配额/认证/429 错误 —— 等待速率窗口重置)、`recent_success`(最近一小时内有完成的运行 —— 在重新运行前等待审查)、`active_pr`(最近的评论中出现 GitHub PR URL —— 先前的 worker 已经打开了 PR)。任务保持在 `ready`;下一个 tick 有另一次启动机会。如果底层条件持续存在,正常的 `consecutive_failures` 熔断器将在 `failure_limit` 次失败后通过 `gave_up` 自动阻塞。 |
| `respawn_guarded` | `{reason}` | 调度器拒绝在本 tick 重新启动此就绪任务。原因:`blocker_auth`(上次失败是配额/认证/429 错误 —— 等待速率窗口重置)和 `recent_success`(最近一小时内有完成的运行 —— 使用显式生命周期转换来请求再次运行)。PR 评论是审计元数据,不会发出此事件。任务保持在 `ready`;下一个 tick 有另一次启动机会。如果底层条件持续存在,正常的 `consecutive_failures` 熔断器将在 `failure_limit` 次失败后通过 `gave_up` 自动阻塞。 |
| `spawn_failed` | `{error, failures}` | 一次启动尝试失败(PATH 缺失、工作区无法挂载等)。计数器递增;任务返回 `ready` 重试。 |
| `protocol_violation` | `{pid, claimer, exit_code}` | Worker 在任务仍处于 `running` 状态时成功退出,通常是因为它回答了问题而没有调用 `kanban_complete` 或 `kanban_block`。调度器还会立即发出 `gave_up` 并自动阻塞,而不是重试。 |
| `gave_up` | `{failures, effective_limit, limit_source, error}` | N 次连续不成功尝试后熔断器触发。任务以最后一个错误自动阻塞。有效限制解析为任务 `max_retries`,然后是调度器 `failure_limit` / `kanban.failure_limit`,然后是内置默认值。 |
Expand Down