From 3c47e1a6934fa45c105b59d4442e36cdaceaa0eb Mon Sep 17 00:00:00 2001 From: James Wiesebron Date: Thu, 25 Jun 2026 08:46:13 -0700 Subject: [PATCH 1/3] refactor(orchestrator): retire in-pod BRC wait arm; orchestrator owns the event loop (#3164) The live proving run on issue-3200 validated orchestrator-owned BRC mode, satisfying #3164's gate. This is the scheduled cleanup of the in-pod path: no ownership flag, no in-pod wait arm, no rollback flag (self-hosting risk is exactly why the live-cycle gate preceded this PR). - Flip ownership unconditional: remove EGG_EVENT_LOOP_OWNER / get_event_loop_owner and both _event_loop_owner helpers. spawn_all always starts the orchestrator event loop and returns [] (no resident pods). - Delete the in-pod wait arm from consensus_wrapper.py: one-shot-only template (no while-loop, no `egg-orch message wait-loop`, no 30s background heartbeat, no wrapper-side idle-budget / fail-streak machinery). The one-shot handler requires EGG_EVENT_ACTION (else exit 64). - Remove the gateway-session heartbeat fan-out (HeartbeatCoordinator + messages route); sessions refresh at spawn time now. - Make HealthMonitor tripwire scoping unconditionally active-Job-scoped (set_active_roles is the only mode; roles with no live Job are idle). - restart_agent is now orchestrator-native: it deletes the role's live one-shot Job(s), resets consensus + health anchor, and lets the event loop respawn rather than spawning a resident pod (HITL decision). - Bump DEFAULT_MAX_PARALLEL_SLICES 1 -> 4: on-demand spawning removed the ~8-resident-container-per-slice cost, so a 4-wide wave is safe. - Update event-pump contract docs (11) + agent rules + the wait-instruction and prompt-sync ratchets: "the wrapper owns all waiting" -> "the orchestrator owns all waiting". --- docs/architecture/brc-memory.md | 5 +- docs/architecture/coordination-state.md | 5 +- .../architecture/on-demand-agent-lifecycle.md | 184 +- docs/architecture/orchestrator.md | 78 +- docs/development/STRUCTURE.md | 2 +- docs/guides/concurrent-execution.md | 12 +- docs/guides/pipeline-health-monitoring.md | 2 +- docs/index.md | 2 +- docs/reference/agent-tools.md | 4 +- docs/reference/agent-wait-patterns.md | 115 +- docs/reference/orchestrator-cli.md | 8 +- orchestrator/concurrent_executor.py | 90 +- orchestrator/consensus_wrapper.py | 1067 ++------- orchestrator/env_config.py | 91 +- orchestrator/event_loop.py | 12 +- orchestrator/health_monitor.py | 62 +- orchestrator/heartbeat.py | 91 +- orchestrator/kubernetes_spawner.py | 21 +- orchestrator/models.py | 16 +- orchestrator/routes/consensus.py | 11 +- orchestrator/routes/event_prompt.py | 18 +- orchestrator/routes/messages.py | 115 +- orchestrator/routes/pipelines.py | 531 ++--- .../tests/golden/event_pump_wrapper.sh.golden | 384 ++++ .../event_pump_wrapper_pod_default.sh.golden | 825 ------- .../tests/test_concurrent_executor.py | 109 +- .../tests/test_concurrent_integration.py | 12 +- orchestrator/tests/test_consensus_wrapper.py | 1965 ++--------------- orchestrator/tests/test_event_loop.py | 44 +- orchestrator/tests/test_health_monitor.py | 361 +-- orchestrator/tests/test_heartbeat.py | 242 +- .../tests/test_infra_error_escalation.py | 9 + orchestrator/tests/test_kubernetes_spawner.py | 11 +- orchestrator/tests/test_messages.py | 429 +--- orchestrator/tests/test_models.py | 12 +- .../test_next_action_iteration_feedback.py | 4 +- .../tests/test_overseer_hitl_integration.py | 3 + .../tests/test_pipeline_failure_path.py | 24 + .../tests/test_prompt_sync_ratchet.py | 2 +- orchestrator/tests/test_restart_agent.py | 1448 +++--------- .../tests/test_two_tier_integration.py | 13 + .../tests/test_wait_instruction_ratchet.py | 54 +- sandbox/agent-config/rules/mission.md | 12 +- sandbox/agent-config/rules/orchestrator.md | 2 +- sandbox/egg_agent_tools/tools/message.py | 17 +- 45 files changed, 1942 insertions(+), 6582 deletions(-) create mode 100644 orchestrator/tests/golden/event_pump_wrapper.sh.golden delete mode 100644 orchestrator/tests/golden/event_pump_wrapper_pod_default.sh.golden diff --git a/docs/architecture/brc-memory.md b/docs/architecture/brc-memory.md index d5ad7178dc..e4e9617595 100644 --- a/docs/architecture/brc-memory.md +++ b/docs/architecture/brc-memory.md @@ -22,8 +22,9 @@ as ephemeral, regardless of any "durable" framing in older design notes. A consensus reviewer's lifecycle before #2908 was a long-running session that held a blocking `egg-orch message wait-loop` between events. Issue -#2908 removed that seam: the wrapper holds the wait, and the agent is -invoked one-shot per actionable event. A stateless one-shot has no working +#2908 removed that seam, and #3164 retired the in-pod wait arm: the +orchestrator now owns the wait and spawns the agent one-shot per +actionable event. A stateless one-shot has no working memory across invocations, so something else has to carry the reviewer's distilled understanding of the codebase, the change under review, and the verdicts already issued on each producer. That something is this file. diff --git a/docs/architecture/coordination-state.md b/docs/architecture/coordination-state.md index 689fd03035..2d781a3dbc 100644 --- a/docs/architecture/coordination-state.md +++ b/docs/architecture/coordination-state.md @@ -227,8 +227,9 @@ quietly trade one for the other. illustrate clause-1 thinking applied to ephemeral coordination state. - [BRC Consensus Wrapper](orchestrator.md#brc-consensus-wrapper) — - the wrapper that holds the wait, performs deterministic sync, and - invokes the per-event agent. + the orchestrator owns the wait and spawns the per-event agent + one-shot (the in-pod wait arm was retired by #3164); the wrapper + around each pod performs deterministic sync at spawn time. - [Reviewer Sync Guide](../../shared/prompts/REVIEWER-SYNC.md) — the agent-facing reviewer contract; its delta-command row was rewritten in #3077 slice 5 from `git fetch` + `git log` instructions to diff --git a/docs/architecture/on-demand-agent-lifecycle.md b/docs/architecture/on-demand-agent-lifecycle.md index 2745e6642d..4da79f1549 100644 --- a/docs/architecture/on-demand-agent-lifecycle.md +++ b/docs/architecture/on-demand-agent-lifecycle.md @@ -1,55 +1,43 @@ # On-Demand Agent Lifecycle -> Event-loop ownership modes, the per-event spawn dedupe contract, failure -> supervision semantics, and the monitor matrix that supports both the legacy -> long-lived-pod path and the orchestrator's on-demand path. Introduced by -> [#3064](https://github.com/jwbron/egg/issues/3064). - -## The #3023 Post-Mortem Constraint - -The first attempt at on-demand agent spawning landed the `EGG_EVENT_LOOP_OWNER` -guard alone. The guard silenced the in-pod event loop, leaving nothing to -service BRC events — deadlock. - -Since #2908 slice-4 deleted the legacy `EGG_BRC_EVENT_PUMP` rollback flag, -there is **no rollback path** to the pre-#3064 behavior. - -The hard constraint: - -- The flag **must default** to the in-pod loop (`pod`). -- The guard and spawner **must land together** (or spawner first) in a single PR. -- The follow-up default-flip is a **separate, gated** step that runs a live - BRC proving run with `EGG_EVENT_LOOP_OWNER=orchestrator` BEFORE committing - to retiring the in-pod path. - -See [Live Proving-Run Procedure](#live-proving-run-procedure) and -[Prepared Follow-Up Issue Body](#prepared-follow-up-issue-body) below. - -## Event-Loop Ownership: Two Modes - -`EGG_EVENT_LOOP_OWNER` ∈ {`pod` (default), `orchestrator`} controls who -runs the consensus event loop and spawns agent pods. - -### `pod` — Long-Lived Pods (Production Default) - -This is the historical mode, unchanged by #3064. The orchestrator spawns the -full agent team up front at phase start. Each agent pod runs the BRC -consensus wrapper in a long-lived in-pod event pump: - -- **Who runs the loop:** the pod's wrapper (`orchestrator/consensus_wrapper.py`). -- **Who spawns:** `concurrent_executor.spawn_all()` → `kubernetes_spawner.spawn_agent_job()` at phase start. -- **Verb→pod mapping:** every role gets a pod. Confirm and complete verbs are - handled agent-free by the wrapper — no agent subprocess is invoked. -- **Lifecycle:** pod lives for the full phase; idle pods reserve CPU/memory - and a gateway session for the duration. -- **Supervision:** in-pod background heartbeat at 30 s interval; idle-budget - alert at `EGG_BRC_IDLE_BUDGET_MIN` (default 30 min) via `OVERSEER_ALERT`. -- **Worktree:** created once at spawn; persists on disk across the pod lifetime. - -### `orchestrator` — On-Demand One-Shot Pods - -Under orchestrator ownership the in-process event loop at the -`concurrent_executor` completion-poll site consumes `_derive_next_action` +> The orchestrator-owned BRC event loop, the per-event spawn dedupe contract, +> failure supervision semantics, and the monitor matrix for the on-demand +> one-shot-pod path. The mechanism was introduced by +> [#3064](https://github.com/jwbron/egg/issues/3064); the in-pod wait arm and +> the `EGG_EVENT_LOOP_OWNER` mode toggle were retired by +> [#3164](https://github.com/jwbron/egg/issues/3164) after the live proving +> run on issue #3200 passed. **Orchestrator ownership is now the only mode.** + +## History & the #3023 Post-Mortem Lesson + +> This section is **lineage**. The current state is single-mode (orchestrator +> ownership); the history below explains why the rollout was staged. + +The first attempt at on-demand agent spawning ([#3023](https://github.com/jwbron/egg/issues/3023)) +landed the `EGG_EVENT_LOOP_OWNER` guard alone. The guard silenced the in-pod +event loop, leaving nothing to service BRC events — deadlock. Since #2908 +slice-4 had already deleted the legacy `EGG_BRC_EVENT_PUMP` rollback flag, +there was no runtime fallback. + +The lesson drove a staged rollout under #3064: + +- The flag (`EGG_EVENT_LOOP_OWNER`) **defaulted** to the in-pod loop (`pod`) + so the mechanism could land dark. +- The guard and spawner **landed together** in #3064. +- The default-flip was a **separate, gated** step: a live BRC proving run with + `EGG_EVENT_LOOP_OWNER=orchestrator` on issue #3200 had to pass the + acceptance checklist (see [Live Proving-Run Procedure](#live-proving-run-procedure), + retained below as history) BEFORE retiring the in-pod path. + +[#3164](https://github.com/jwbron/egg/issues/3164) completed that sequence: +the proving run passed, the default flipped, and the in-pod wait arm + the +`EGG_EVENT_LOOP_OWNER` flag were removed. There is **no rollback flag** — the +only regression path is `git revert` of the #3164 change. + +## Event-Loop Ownership: Orchestrator-Only + +The orchestrator owns the consensus event loop. The in-process event loop at +the `concurrent_executor` completion-poll site consumes `_derive_next_action` per role and spawns an agent pod **only** when that role has an actionable event (`propose` | `ack` | `nack`). Each pod handles a single event and exits. @@ -73,8 +61,6 @@ exits. ┌───────────────────────────────┐ │ one-shot agent pod │ │ │ - │ EGG_EVENT_LOOP_OWNER │ - │ = orchestrator │ │ EGG_EVENT_ACTION │ │ = propose|ack|nack │ │ EGG_EVENT_DEDUPE_KEY │ @@ -88,6 +74,11 @@ exits. └─────────────────────────────┘ ``` +The wrapper around each pod is **one-shot-only**: it requires `EGG_EVENT_ACTION` +to be set, handles exactly that single event, then exits. There is no in-pod +wait-loop between events and no 30 s background heartbeat subshell — both were +retired in #3164. + **Verb → Pod mapping:** | Verb | Action | Pod spawned? | @@ -100,8 +91,8 @@ exits. | `wait` | No peer action | ❌ Nothing | **Key design decisions:** -- `confirm`/`complete` never reach a pod — the wrapper already handles them - agent-free, and the orchestrator-side path mirrors that. +- `confirm`/`complete` never reach a pod — the orchestrator handles them + agent-free. - The orchestrator's poll interval is env-tunable (default **5 s**). ## Dedupe-Key Contract @@ -162,9 +153,9 @@ values, no fork. - Legitimate BRC protocol steps of any kind **AGENT_FAILED engagement:** Producer propose-arm exhaustion engages the existing -`AGENT_FAILED` path (#2806) relocated for orchestrator mode — same phase-level -HITL escalation as pod mode. Wrapper-side #2806 code is untouched and will be -deleted in the follow-up cleanup PR. +`AGENT_FAILED` path (#2806) on the orchestrator side — phase-level HITL +escalation. (The wrapper-side #2806 in-pod code was removed in #3164 when the +in-pod wait arm was retired.) ## Worktree Re-attach & Session Reuse (Hot-Path Latency) @@ -184,31 +175,39 @@ Under orchestrator ownership the worktree becomes a hot path. **Session reuse:** - Re-register against the gateway only when no live session exists or the token has aged out. -- Session teardown moves to **phase end** or **streak exhaustion** in orchestrator - mode. -- Pod-mode session lifecycle unchanged. +- Session teardown happens at **phase end** or **streak exhaustion**. +- The gateway session is refreshed **at spawn time**, not via a between-events + heartbeat fan-out (the 30 s background heartbeat was retired in #3164). **Latency budget:** p50 spawn→invoke **< 60 s** (asserted in a simulated-clock test). -## Monitor Matrix (Tripwire × Ownership Mode) +## Monitor Matrix -| Tripwire | `pod` mode | `orchestrator` mode | -|----------|-----------|---------------------| -| "Role has no pod" | Trip (pod died) | **Normal** — never alerts | -| Heartbeat timeout (120 s default, 600 s implement) | Active | **Active only while a Job is running** | -| Container exit | Trip on unexpected exit | **Trips only if a Job was active** | -| Silent mid-event pod | — | **Trips** (one-shot pod that goes silent mid-invocation triggers) | -| Idle-budget alert (`EGG_BRC_IDLE_BUDGET_MIN`) | In-pod alert (wrapper-side) | **Convergence-stall judgment** from tracker timestamps — same knob, same anomaly name — orchestrator-side | +Orchestrator ownership is the only mode (#3164). Tripwires are active **only +while a role's one-shot Job is live**; between dispatched events a role has no +running pod, and that silence is normal. -**HeartbeatCoordinator mode guard:** In orchestrator mode, session refresh happens -at spawn (not via a background heartbeat subprocess). Absent senders between events -trip nothing. +| Tripwire | Behaviour | +|----------|-----------| +| "Role has no pod" | **Normal** — never alerts (between-events silence is expected) | +| Heartbeat timeout (120 s default, 600 s implement) | **Active only while a Job is running** | +| Container exit | **Trips only if a Job was active** | +| Silent mid-event pod | **Trips** — a one-shot pod that goes silent mid-invocation (Job active, no heartbeat) alerts normally | +| Idle-budget alert (`EGG_BRC_IDLE_BUDGET_MIN`) | **Convergence-stall judgment** from tracker timestamps, orchestrator-side — same knob, same anomaly name as the retired in-pod budget check | + +**HeartbeatCoordinator:** session refresh happens at spawn time (not via a +background heartbeat subprocess). Absent senders between events trip nothing. +The active-Job set is refreshed every poll tick by the event loop via +`set_active_roles(roles)`, so coverage is current. ## Live Proving-Run Procedure -Before the production default can be flipped from `pod` to `orchestrator`, -a live BRC pipeline must run through all phases with -`EGG_EVENT_LOOP_OWNER=orchestrator` and pass this acceptance checklist: +> **History.** This checklist gated the #3164 default-flip. The proving run +> ran on issue #3200 and passed; the default flipped and the in-pod arm was +> retired. Retained for the acceptance criteria of record. + +A live BRC pipeline ran through all phases with the orchestrator owning the +event loop and passed this acceptance checklist: 1. **All phases converge** — BRC consensus completes in refine, plan, implement (all slices), and PR; no phase stalls or deadlocks. @@ -222,40 +221,37 @@ a live BRC pipeline must run through all phases with 4. **Latency budget held** — measured p50 spawn→invoke latency across all events ≤ 60 s. -## Prepared Follow-Up Issue Body +## Follow-Up Sequence (completed by #3164) -The follow-up issue is filed **immediately post-merge** (manual action -referenced from the PR description) and encodes the operator-mandated -sequence: +> **History.** The #3064 PR filed [#3164](https://github.com/jwbron/egg/issues/3164) +> post-merge to carry out the operator-mandated retirement sequence. #3164 +> has since landed; the sequence below is the record of what it did. -1. **Live BRC proving run** with `EGG_EVENT_LOOP_OWNER=orchestrator` to - pass the [acceptance checklist](#live-proving-run-procedure). -2. **Flip the default** to `orchestrator`. -3. **One cleanup PR** deleting: - - The in-pod wait arm (≈`consensus_wrapper.py:379` event-pump loop) - - The background heartbeat subprocess (≈`209-230`) - - Wrapper-side #3138 streak code (≈`897-901`) - - Wrapper-side #2806 failure-code path (≈`134`+) - - The `EGG_EVENT_LOOP_OWNER` flag itself - - **End state: no dead/deprecated code.** +The sequence #3164 executed: -Filing the issue is an immediate post-merge manual step. The follow-up PR -body should be a copy-paste from this section. **Issue [#3164](https://github.com/jwbron/egg/issues/3164) -is reserved for this purpose** (the operator directed filing during the -plan phase gate review). +1. **Live BRC proving run** (issue #3200) with the orchestrator owning the + event loop, passing the [acceptance checklist](#live-proving-run-procedure). +2. **Flipped the default** to orchestrator ownership. +3. **Retired the in-pod path**, deleting: + - The in-pod wait arm (the `consensus_wrapper.py` event-pump loop) + - The 30 s background heartbeat subprocess + - Wrapper-side #3138 streak code + - Wrapper-side #2806 failure-code path + - The `EGG_EVENT_LOOP_OWNER` flag itself (and `get_event_loop_owner`) + - **End state: no dead/deprecated code; orchestrator ownership is the only mode.** ## References - [#3064](https://github.com/jwbron/egg/issues/3064) — orchestrator-owned event loop - [#3023](https://github.com/jwbron/egg/issues/3023) — scrapped first attempt (post-mortem) -- [#3164](https://github.com/jwbron/egg/issues/3164) — flip follow-up issue (post-merge filing) +- [#3164](https://github.com/jwbron/egg/issues/3164) — default-flip + in-pod-arm retirement (orchestrator ownership is the only mode) - [#2908](https://github.com/jwbron/egg/issues/2908) — BRC consensus wrapper (foundation) - [#3138](https://github.com/jwbron/egg/issues/3138) — streak/backoff supervision semantics - [#2806](https://github.com/jwbron/egg/issues/2806) — AGENT_FAILED path - [#2761](https://github.com/jwbron/egg/issues/2761) — tracker rebuild from message store - `orchestrator/supervision_policy.py` — shared streak constants (NEW, slices 3–6) - `orchestrator/event_loop.py` — orchestrator-side event loop + dedupe + supervision (NEW, slices 2–5) -- `orchestrator/consensus_wrapper.py` — wrapper template (one-shot arm, ~line 379 wait-loop) +- `orchestrator/consensus_wrapper.py` — wrapper template (one-shot-only per-event arm; the in-pod wait-loop was retired in #3164) - `orchestrator/kubernetes_spawner.py` — one-shot Job spawning (NEW entry, slices 2, 4) - `orchestrator/health_monitor.py` — lifecycle-aware tripwires - `orchestrator/heartbeat.py` — HeartbeatCoordinator mode guard diff --git a/docs/architecture/orchestrator.md b/docs/architecture/orchestrator.md index fa20a83379..b4a3b28f05 100644 --- a/docs/architecture/orchestrator.md +++ b/docs/architecture/orchestrator.md @@ -751,8 +751,21 @@ POST /api/v1/pipelines/{pipeline_id}/signal > event-pump path is the **only** consensus-wrapper path — the > legacy `_CONSENSUS_WRAPPER_TEMPLATE`, the `_RECOVERY_SYSTEM_PROMPT`, > the SSE `consensus.reached` machinery, and the -> `MAX_CONSENSUS_RESTARTS = 3` cap are gone. The wait-side companion -> is [agent-wait-patterns §10](../reference/agent-wait-patterns.md#10-brc-consensus-wrapper-event-pump-model). +> `MAX_CONSENSUS_RESTARTS = 3` cap are gone. +> +> **[#3164](https://github.com/jwbron/egg/issues/3164) retired the in-pod +> wait arm.** After the live proving run on issue #3200 passed, #3164 +> removed the in-pod `egg-orch message wait-loop` between events, the 30 s +> background heartbeat subshell, and the `EGG_EVENT_LOOP_OWNER` ownership +> flag (there is no "pod vs orchestrator" mode toggle anymore — orchestrator +> ownership is the only mode). The **orchestrator** (`orchestrator/event_loop.py`) +> now owns the BRC event loop: it derives the next action in-process and +> spawns a one-shot pod per actionable event; the wrapper around each pod is +> one-shot-only (it handles exactly the single `EGG_EVENT_ACTION` it was +> spawned for, then exits). The section below describes the historical +> in-pod loop for lineage; the current-state ownership is orchestrator-side. +> The wait-side companion is +> [agent-wait-patterns §10](../reference/agent-wait-patterns.md#10-brc-consensus-wrapper-event-pump-model). ### Why the wrapper drives the loop @@ -838,19 +851,22 @@ The conditional inclusion is pinned by a snapshot test in `orchestrator/tests/test_consensus_wrapper.py` so the regression that spawned this whole lineage cannot land again silently. -### Wrapper-side heartbeat (#2036 migration completed in slice-4) +### Wrapper-side heartbeat (#2036 migration completed in slice-4; retired by #3164) -Heartbeats are emitted from a **background subshell inside the -wrapper bash** that fires `egg-orch message heartbeat` every 30 s -while `egg-orch message wait-loop` is blocking. The wrapper owns -this because the agent is one-shot per event: there is no in-pod -loop left to emit heartbeats between events, so the wrapper is the -only process alive across the full BRC cycle. The pre-#2908 agent-side -heartbeat path in +**Retired by [#3164](https://github.com/jwbron/egg/issues/3164).** +Historically (post-#2036, pre-#3164) heartbeats were emitted from a +**background subshell inside the wrapper bash** that fired +`egg-orch message heartbeat` every 30 s while `egg-orch message wait-loop` +was blocking. That subshell existed because the wrapper held a long-lived +in-pod wait between events. The pre-#2908 agent-side heartbeat path in `sandbox/egg_agent_tools/handlers/message.py::message_wait_loop` was **deleted in slice-4 task-4-2** — agents no longer self-emit -`WAITING_FOR_EVENT` while blocked, and the overseer's stall detector -consumes the wrapper-side stream instead. +`WAITING_FOR_EVENT` while blocked. #3164 then removed the in-pod wait arm +and the 30 s background heartbeat subshell entirely: the orchestrator now +spawns a one-shot pod per actionable event, so there is no long-lived +in-pod process to heartbeat between events, and the overseer judges +convergence stall from tracker timestamps orchestrator-side (see +[Monitor Matrix in on-demand-agent-lifecycle.md](on-demand-agent-lifecycle.md#monitor-matrix)). **`slice_id` propagation invariant.** The wrapper-side heartbeat payload **must** include `slice_id` sourced from the @@ -900,18 +916,17 @@ WS7-observed 10–13 min idle ceiling on real BRC phases). |---------------------------|-----------| | default `30` (minutes) | At budget threshold, wrapper emits `mcp__progress__overseer_alert` (anomaly `stuck-phase-transition`, priority `high`) **and keeps blocking**. At `2 ×` budget, the alert priority escalates and the wrapper still keeps blocking. Idleness is **not** a FAILED transition. | -**Slice-5 re-homing (#3064):** In orchestrator mode -(`EGG_EVENT_LOOP_OWNER=orchestrator`) the in-pod `check_idle_budget` -no longer runs. Instead the orchestrator event loop judges convergence -stall from tracker timestamps: when a role's derived actionable event +**Re-homed to the orchestrator (#3064 slice-5, in-pod arm retired by #3164):** +The in-pod `check_idle_budget` no longer runs — #3164 retired the in-pod +wait arm. The orchestrator event loop judges convergence stall from tracker +timestamps: when a role's derived actionable event (`propose`|`ack`|`nack`) has been pending longer than `EGG_BRC_IDLE_BUDGET_MIN` minutes with no BRC-bus activity, the event -loop raises the same `stuck-phase-transition` anomaly the in-pod -budget check uses. The threshold is single-sourced via -`event_loop.get_idle_budget_minutes()` which reads the same env var; +loop raises the `stuck-phase-transition` anomaly. The threshold is sourced +via `event_loop.get_idle_budget_minutes()` which reads the same env var; the anomaly name is single-sourced via -`consensus_wrapper.EVENT_PUMP_IDLE_BUDGET_ANOMALY` so orchestrator-side -and in-pod alerts are classified identically by the overseer. +`consensus_wrapper.EVENT_PUMP_IDLE_BUDGET_ANOMALY` so the alert is +classified consistently by the overseer. The slice-2/-3 era also exposed an `EGG_BRC_EVENT_PUMP=false` escape to the legacy capped-restart wrapper; that escape is gone after @@ -980,6 +995,15 @@ real behaviour. The current test surface: ### Rollback plan +> **No in-pod rollback flag after [#3164](https://github.com/jwbron/egg/issues/3164).** +> #3164 retired the in-pod wait arm and the `EGG_EVENT_LOOP_OWNER` +> ownership flag with **no rollback flag** — there is no runtime toggle +> back to a "pod" mode, because the in-pod wait loop and 30 s background +> heartbeat no longer exist. The only regression path is `git revert` of +> the #3164 change (and, for the deeper event-pump infrastructure, the +> #2908 slice sequence below). The orchestrator-owned event loop +> (`orchestrator/event_loop.py`) is the only mode after #3164. + The supported regression path if production traffic shows event-pump regression is **`git revert` of the slice-4, slice-3, slice-2, and slice-1 merge commits** (in reverse-merge order). The @@ -1056,7 +1080,7 @@ above). It assembles, in order: |----------|---------|--------|-------| | Top | Role banner + one-line event description | `role` + `event_payload.kind` | A few hundred bytes; identifies the producer/reviewer side of the dispatch. | | Top | Task & operator directives (#3123/#3163) | The contract's `task_description`, read from the worktree contract file via the pod-inherited `EGG_PIPELINE_ID` / `EGG_ISSUE_NUMBER`; populated for all pipeline types since #3163 (issue anchor + submit description); omitted only when the contract carries no task statement and no issue identity | ≤ 4 KB (`TASK_DESCRIPTION_MAX_CHARS`), truncated with a pointer to `mcp__sdlc__show_contract`. Pushes the operator's submit-time directives into every invocation instead of relying on the agent pulling them per the rules file. | -| Top | Operator feedback steering this phase (#3231) | The current phase execution's `operator_directives` (chronological) threaded onto the event_payload by the `next-action` route from persisted pipeline state. Carried on **both** the `propose` arm (kicked-back producer's re-run — directives **plus** the latest `iteration_history` verdict/NACK summary, #2795) and the `ack` arm (reviewer re-reviewing the producer's directive-driven change — directives only, no producer scorecard). The `audience` tag selects producer vs. reviewer framing. Rendered in full for the most-recent directive, one line each for earlier ones. | ≤ 4 KB (`ITERATION_FEEDBACK_MAX_CHARS`), truncated with a pointer to `egg-orch brc get-state`; also a second-stage envelope-truncation candidate after NACKs. Surfaced only under `EGG_EVENT_LOOP_OWNER=orchestrator` — the in-pod path already carries #2795's iteration context for both producers and reviewers. Omitted when there is no kickback (golden-stable). Makes an unchanged producer re-propose after `request_changes` a defect, and stops reviewers NACKing a directive-driven change back to a stale rubric. | +| Top | Operator feedback steering this phase (#3231) | The current phase execution's `operator_directives` (chronological) threaded onto the event_payload by the `next-action` route from persisted pipeline state. Carried on **both** the `propose` arm (kicked-back producer's re-run — directives **plus** the latest `iteration_history` verdict/NACK summary, #2795) and the `ack` arm (reviewer re-reviewing the producer's directive-driven change — directives only, no producer scorecard). The `audience` tag selects producer vs. reviewer framing. Rendered in full for the most-recent directive, one line each for earlier ones. | ≤ 4 KB (`ITERATION_FEEDBACK_MAX_CHARS`), truncated with a pointer to `egg-orch brc get-state`; also a second-stage envelope-truncation candidate after NACKs. Surfaced via the orchestrator event loop (the only mode post-#3164). Omitted when there is no kickback (golden-stable). Makes an unchanged producer re-propose after `request_changes` a defect, and stops reviewers NACKing a directive-driven change back to a stale rubric. | | Middle | NACK payload (per-reviewer NACK with `reason` + `artifact_refs`) | `orchestrator/peer_consensus.py` `_open_nacks_barrier_response.nacks[]` (line numbers come from the slice-3 contract spec and are drift-prone — prefer the function-name reference; the function span is around lines 949–1046 in practice) — the same envelope the producer sees on the aggregated-NACK 409 (see [§10.6](../reference/agent-wait-patterns.md#106-409-stale_version--aggregated-nack-are-event-pump-signals-not-transient-errors)). | One section per reviewer that NACKed the current version; rendered with reason text + artifact references. | | Middle | Single expected action | `event_payload.kind` | A few hundred bytes; states whether the agent should review, fix, confirm, etc. | | Tail | Per-producer git-log delta | `git log {last_reviewed_commit_sha}..HEAD --not origin/{base_branch} -p` ← `last_reviewed_commit_sha` from the [BRC memory file](brc-memory.md) | Scaled by actual change size; **NOT** counted against the 10 KB envelope. | @@ -1217,9 +1241,9 @@ spawn. | Removed from preamble | Why | |-----------------------|-----| -| Producer Lifecycle step 4 wait-loop plumbing | Wrapper holds the wait. | -| Producer Lifecycle step 6 STAY-ALIVE loop | Wrapper loops back to `brc next-action` instead. | -| Cursor / `--since` threading guidance | Cursor threading is automatic and wrapper-internal under the event-pump path. | +| Producer Lifecycle step 4 wait-loop plumbing | The orchestrator owns the wait (post-#3164). | +| Producer Lifecycle step 6 STAY-ALIVE loop | The orchestrator derives `brc next-action` and re-spawns per event instead. | +| Cursor / `--since` threading guidance | Cursor threading is automatic and not agent-facing under the event-pump path. | | Pre-confirm-wait foot-gun guidance (anti-pattern 5) | The wrapper's conditional filter (see [Wait-filter construction (pre-confirm vs post-confirm)](#wait-filter-construction-pre-confirm-vs-post-confirm)) prevents the regression at the source. | | Kept in preamble | Why | @@ -1357,8 +1381,8 @@ if is_orchestrator_mode(): | `EGG_ORCH_STACKED_PR_RECONCILER_INTERVAL_SECONDS` | Slice-DAG: stacked-PR reconciler polling cadence for orphaned child PRs (#2137) | `30.0` | | `EGG_BRC_EVENT_PUMP` | **Removed in [#2908](https://github.com/jwbron/egg/issues/2908) slice-4 task-4-2.** During the slice-2/-3 rollout this flag selected between the legacy `_CONSENSUS_WRAPPER_TEMPLATE` (`false`) and the new `_EVENT_PUMP_WRAPPER_TEMPLATE` (`true`). Slice-4 deleted the legacy template, the surrounding selector logic, and the env var read itself — the orchestrator no longer consults this variable. Operators that referenced it in helm values / pod-spec env can drop the row. The supported regression path is `git revert` of slices 1–3 / slice-4 in reverse-merge order (see [Rollback plan](#rollback-plan)); reverting slice-4 restores the env var alongside the legacy template. | n/a (removed) | | `EGG_BRC_IDLE_BUDGET_MIN` | BRC consensus wrapper idle / no-progress safety budget in minutes ([#2908](https://github.com/jwbron/egg/issues/2908)). Replaced the legacy 3-restart FAIL cap with an overseer-alert escalation that does not transition the pipeline to FAILED. At budget threshold the wrapper emits `mcp__progress__overseer_alert` (anomaly `stuck-phase-transition`, priority `high`) and keeps blocking; at `2 ×` budget the priority escalates and the wrapper still keeps blocking. Default 30 min is well above the WS7-observed 10–13 min idle ceiling on real BRC phases. In orchestrator mode ([#3064](https://github.com/jwbron/egg/issues/3064) slice-5), the orchestrator event loop also reads this value for its convergence-stall check (re-homed from the in-pod wrapper) — see [Idle / no-progress safety budget](#idle--no-progress-safety-budget). | `30` | -| `EGG_EVENT_LOOP_OWNER` | BRC event-loop ownership mode ([#3064](https://github.com/jwbron/egg/issues/3064) slice-2). `pod` (default) — each agent container runs the in-pod event-pump wait-loop in `consensus_wrapper.py`, matching the pre-#3064 behaviour. `orchestrator` — the orchestrator owns the BRC event loop and spawns a one-shot pod per actionable event (`propose`/`ack`/`nack`); `confirm`/`complete` execute orchestrator-side with no pod. Spawns are deduped on a SHA-256 key so orchestrator restarts never double-spawn. **An unrecognised value raises `ValueError` loudly** (no silent fallback — a misconfiguration would either deadlock BRC or duplicate pods). Read by `orchestrator/env_config.py::get_event_loop_owner`. | `pod` | -| `EGG_EVENT_LOOP_POLL_INTERVAL_SECONDS` | Orchestrator-owned BRC event loop poll cadence in seconds ([#3064](https://github.com/jwbron/egg/issues/3064) slice-2). Only effective when `EGG_EVENT_LOOP_OWNER=orchestrator`. Malformed or non-positive values fall back to the default with a warning. Read by `orchestrator/event_loop.py::get_event_loop_poll_interval`. | `5.0` | +| `EGG_EVENT_LOOP_OWNER` | **Removed in [#3164](https://github.com/jwbron/egg/issues/3164).** During the #3064 rollout this flag selected `pod` (in-pod event-pump wait-loop in `consensus_wrapper.py`) vs `orchestrator` (orchestrator-owned BRC event loop). #3164 retired the in-pod wait arm — orchestrator ownership is now the only mode — and removed `get_event_loop_owner` from `orchestrator/env_config.py`. Operators that set this in helm values / pod-spec env can drop the row; it is no longer read. | n/a (removed) | +| `EGG_EVENT_LOOP_POLL_INTERVAL_SECONDS` | Orchestrator-owned BRC event loop poll cadence in seconds ([#3064](https://github.com/jwbron/egg/issues/3064) slice-2). Always effective post-#3164 (orchestrator ownership is the only mode). Malformed or non-positive values fall back to the default with a warning. Read by `orchestrator/event_loop.py::get_event_loop_poll_interval`. | `5.0` | ### Constants diff --git a/docs/development/STRUCTURE.md b/docs/development/STRUCTURE.md index c7c8fc5e45..c292bab279 100644 --- a/docs/development/STRUCTURE.md +++ b/docs/development/STRUCTURE.md @@ -112,7 +112,7 @@ orchestrator/ ├── action_guards.py # Formal BRC state machine action guards (preconditions for propose/ack/nack/confirm/withdraw) ├── approval_matrix.py # Per-reviewer ACK/NACK matrix for BRC consensus ├── attestation_schemas.py # Attestation payload validation for BRC proposals -├── consensus_wrapper.py # Shell wrapper template: deterministic event-pump loop (sole path since slice-4; legacy capped-restart template and EGG_BRC_EVENT_PUMP flag deleted); contains dormant one-shot arm (#3064 slice-1) spliced in when EGG_EVENT_LOOP_OWNER=orchestrator +├── consensus_wrapper.py # Shell wrapper template: one-shot-only per-event arm (#3164 retired the in-pod wait-loop + 30s heartbeat; legacy capped-restart template and EGG_BRC_EVENT_PUMP flag deleted in #2908 slice-4). The orchestrator (event_loop.py) owns the BRC loop and spawns a one-shot pod per actionable event (EGG_EVENT_ACTION) ├── dag_visualizer.py # ASCII DAG visualization for pipeline status ├── decision_queue.py # HITL decision queue ├── events.py # Event bus for pipeline events diff --git a/docs/guides/concurrent-execution.md b/docs/guides/concurrent-execution.md index fa3008a839..92db08de59 100644 --- a/docs/guides/concurrent-execution.md +++ b/docs/guides/concurrent-execution.md @@ -71,11 +71,11 @@ Each agent is registered in the peer consensus tracker before spawning begins. ## Consensus Wrapper -> **[#2908](https://github.com/jwbron/egg/issues/2908) replaced the model-driven restart loop with a deterministic event-pump wrapper across slices 1–4.** The wrapper is now the sole loop driver — the agent is one-shot per actionable BRC event, the wrapper holds the wait between events, and heartbeats + gateway-session keep-alive ride on wrapper-side subshells rather than the in-pod `message_wait_loop`. The 3-restart `MAX_CONSENSUS_RESTARTS` FAIL cap was replaced by an idle/no-progress overseer-alert budget that never transitions the pipeline to FAILED. The legacy `_CONSENSUS_WRAPPER_TEMPLATE`, the `_RECOVERY_SYSTEM_PROMPT`, the SSE `consensus.reached` machinery, the `MAX_CONSENSUS_RESTARTS` constant, and the `EGG_BRC_EVENT_PUMP=false` escape hatch were deleted in slice-4 — the supported regression path is `git revert` of the slice-4 / slice-3 / slice-2 / slice-1 merge commits in reverse-merge order (see [Orchestrator Architecture — Rollback plan](../architecture/orchestrator.md#rollback-plan)). See [Orchestrator Architecture — BRC Consensus Wrapper](../architecture/orchestrator.md#brc-consensus-wrapper) for the deterministic-loop semantics, the wait-filter construction, the wrapper-side heartbeat + keep-alive subshells, the idle-budget escalation table, and the rollback plan. The wait-side companion is [agent-wait-patterns §10](../reference/agent-wait-patterns.md#10-brc-consensus-wrapper-event-pump-model). +> **[#2908](https://github.com/jwbron/egg/issues/2908) replaced the model-driven restart loop with a deterministic event-pump across slices 1–4; [#3164](https://github.com/jwbron/egg/issues/3164) then retired the in-pod wait arm so the orchestrator owns the loop.** The agent is one-shot per actionable BRC event. Post-#3164 the orchestrator derives the next action in-process (`egg-orch brc next-action`) and spawns a one-shot pod per event — there is no more in-pod wait loop, no 30 s background heartbeat subshell, and no `EGG_EVENT_LOOP_OWNER` mode toggle (orchestrator ownership is the only mode). The wrapper that still wraps each pod is one-shot-only: it handles exactly the single `EGG_EVENT_ACTION` it was spawned for, then exits. The 3-restart `MAX_CONSENSUS_RESTARTS` FAIL cap was replaced by an idle/no-progress overseer-alert budget that never transitions the pipeline to FAILED. See [Orchestrator Architecture — BRC Consensus Wrapper](../architecture/orchestrator.md#brc-consensus-wrapper) for the orchestrator-owned event-loop semantics, the wait-filter construction, the idle-budget escalation table, and the supervision model. The wait-side companion is [agent-wait-patterns §10](../reference/agent-wait-patterns.md#10-brc-consensus-wrapper-event-pump-model). -All concurrent agent containers are wrapped with a shell script defined in `orchestrator/consensus_wrapper.py`. The wrapper is the deterministic BRC loop driver: it polls `egg-orch brc get-state` / `egg-orch brc next-action` for sequencing, blocks in `egg-orch message wait-loop` between actionable events, and invokes the agent one-shot per `INVOKE` event with a prompt composed by `compose_event_prompt` (slice-3). +All concurrent agent containers are wrapped with a shell script defined in `orchestrator/consensus_wrapper.py`. Post-[#3164](https://github.com/jwbron/egg/issues/3164) the **orchestrator** is the deterministic BRC loop driver: it derives the next action in-process from `egg-orch brc next-action` / `egg-orch brc get-state` and spawns a one-shot pod per actionable event. The wrapper around each pod is one-shot-only — it composes the per-event prompt via `compose_event_prompt`, invokes the agent for exactly the single `EGG_EVENT_ACTION` it was spawned for, and exits. It no longer blocks in `egg-orch message wait-loop` between events and no longer runs a background heartbeat subshell; those mechanics were retired with the in-pod wait arm in #3164. -**How it works:** +**How it works** (the in-pod loop below is the historical pre-[#3164](https://github.com/jwbron/egg/issues/3164) path, retained for context; #3164 retired the in-pod wait arm and moved the loop to the orchestrator, which derives the next action in-process and spawns a one-shot pod per event): 1. The wrapper starts two background subshells: a heartbeat emitter (every 30 s while a `wait-loop` is blocking) and the implicit gateway-session keep-alive that rides on every accepted heartbeat. Both surfaces moved here from the pre-#2908 agent-side `message_wait_loop` (see [Wrapper-side heartbeat (#2036)](../architecture/orchestrator.md#wrapper-side-heartbeat-2036-migration-completed-in-slice-4) and [Wrapper-side gateway-session keep-alive (#2451)](../architecture/orchestrator.md#wrapper-side-gateway-session-keep-alive-2451-migration-completed-in-slice-4)). 2. The wrapper polls `egg-orch brc get-state --json`. If `role_complete` is true, it calls `egg-orch consensus confirmed` and exits with code 0. @@ -84,9 +84,9 @@ All concurrent agent containers are wrapped with a shell script defined in `orch 5. If the agent exits non-zero (transient signal-class crash, exit 1, or anything else), the wrapper increments `AGENT_FAIL_STREAK`, sleeps 1 s, and resumes the loop — there is no per-crash restart, no exponential backoff, and no exit-code branching on the `propose|ack|nack` arm. The `is_transient_crash` / `is_buffer_overflow` / `is_startup_failure` helpers are still defined in the wrapper bash template but are **not invoked** today; they are retained as named placeholders against a possible future classifier-gated fast-fail path (see `orchestrator/consensus_wrapper.py` L145–150). 6. The pre-#2908 `MAX_CONSENSUS_RESTARTS = 3` cap and the `_RECOVERY_SYSTEM_PROMPT` recovery-restart cycle no longer exist. The wrapper does not propagate non-zero agent exits — they accrue against the **idle / no-progress safety budget** (`EGG_BRC_IDLE_BUDGET_MIN`, default 30 minutes), which is the sole operator-visible escalation surface. At threshold and `2 ×` threshold the wrapper emits an `OVERSEER_ALERT`; the pipeline is never transitioned to FAILED by this path. -**Key design principle:** Agents must **explicitly** participate in consensus. The wrapper never auto-signals `READY` on behalf of an agent — it dispatches the agent to handle each actionable event and lets the agent issue ACK / NACK / PROPOSE / CONFIRM verdicts itself. Sequencing is the wrapper's job; *judgment* (what to review, what to fix, when to re-propose) stays with the model. +**Key design principle:** Agents must **explicitly** participate in consensus. The orchestrator never auto-signals `READY` on behalf of an agent — it spawns the agent to handle each actionable event and lets the agent issue ACK / NACK / PROPOSE / CONFIRM verdicts itself. Sequencing is the orchestrator's job; *judgment* (what to review, what to fix, when to re-propose) stays with the model. -**Design intent — the wrapper is the loop, not a safety net.** Before #2908 the wrapper existed as a fallback that re-spawned the agent on premature exit (e.g., context exhaustion) and the agent was expected to hold a long-running BRC wait between events. After #2908 the wrapper *is* the BRC loop driver — clean per-event exits are the steady-state contract, not an edge case. The orchestrator detects consensus and sends SIGTERM to terminate containers when the role is complete; the agent only exits because it finished its event handler or because consensus closed. There is no expensive context-reload-on-restart path. +**Design intent — the orchestrator is the loop, not a safety net.** Before #2908 the wrapper existed as a fallback that re-spawned the agent on premature exit (e.g., context exhaustion) and the agent was expected to hold a long-running BRC wait between events. #2908 made the loop deterministic, and #3164 moved it to the orchestrator: the orchestrator *is* the BRC loop driver — clean per-event exits are the steady-state contract, not an edge case. The orchestrator detects consensus and sends SIGTERM to terminate containers when the role is complete; the agent only exits because it finished its event handler or because consensus closed. There is no expensive context-reload-on-restart path. **Configuration:** @@ -104,7 +104,7 @@ Agents communicate with each other during concurrent execution via the orchestra ### How to Wait -**Agents don't wait — the consensus wrapper does.** Since [#2908](https://github.com/jwbron/egg/issues/2908) every blocking read on the message bus belongs to the wrapper tier: the wrapper blocks in `egg-orch message wait-loop` between actionable events (see [Consensus Wrapper](#consensus-wrapper) above) and invokes the agent one-shot per event. The agent handles the single event in its prompt and exits naturally; it must never run `egg-orch message wait` / `wait-loop`, an outer poll loop, or a `sleep` — the wrapper owns the wait and the heartbeat, and an agent-tier wait double-waits against it while holding the SDK turn open. The agent-facing contract is stated in the per-event prompt itself (`orchestrator/routes/event_prompt.py`); the wait-side mechanics — the wrapper's `wait-loop` idiom and per-state filters, the five anti-patterns the pump retired, the `egg-orch message wait` exit codes, the `HEARTBEAT` schema, and the `EGG_MESSAGE_POLL_MAX_WAIT` ↔ gateway-Squid coupling — are in [Agent Wait Patterns](../reference/agent-wait-patterns.md). +**Agents don't wait — the orchestrator does.** Since [#2908](https://github.com/jwbron/egg/issues/2908) every blocking read on the message bus belongs outside the agent tier; [#3164](https://github.com/jwbron/egg/issues/3164) moved it to the orchestrator: the orchestrator derives the next BRC event in-process (`egg-orch brc next-action`) and spawns the agent one-shot per actionable event (see [Consensus Wrapper](#consensus-wrapper) above). There is no in-pod wait loop. The agent handles the single event in its prompt and exits naturally; it must never run `egg-orch message wait` / `wait-loop`, an outer poll loop, or a `sleep` — the orchestrator owns the wait, and an agent-tier wait double-waits against it while holding the SDK turn open. The agent-facing contract is stated in the per-event prompt itself (`orchestrator/routes/event_prompt.py`); the wait-side mechanics — the wrapper's `wait-loop` idiom and per-state filters, the five anti-patterns the pump retired, the `egg-orch message wait` exit codes, the `HEARTBEAT` schema, and the `EGG_MESSAGE_POLL_MAX_WAIT` ↔ gateway-Squid coupling — are in [Agent Wait Patterns](../reference/agent-wait-patterns.md). The wrapper constructs the `--for` filter set conditionally per BRC state ([wait-filter construction](../architecture/orchestrator.md#wait-filter-construction-pre-confirm-vs-post-confirm)): diff --git a/docs/guides/pipeline-health-monitoring.md b/docs/guides/pipeline-health-monitoring.md index 27bc23611c..0955419157 100644 --- a/docs/guides/pipeline-health-monitoring.md +++ b/docs/guides/pipeline-health-monitoring.md @@ -173,7 +173,7 @@ The `HealthMonitor` tracks the current pipeline phase via `set_current_phase()`, ### Orchestrator-Mode Alert Suppression -When the orchestrator owns the BRC event loop (`EGG_EVENT_LOOP_OWNER=orchestrator`, [#3064](https://github.com/jwbron/egg/issues/3064) slice-5), agent pods are short-lived one-shot Jobs rather than long-running containers. Between dispatched events a role has no running pod — and that is normal. The health monitor and heartbeat coordinator are both placed in **orchestrator mode** by the `ConcurrentPhaseExecutor` at event-loop setup: +The orchestrator owns the BRC event loop ([#3064](https://github.com/jwbron/egg/issues/3064) slice-5; the in-pod wait arm and the `EGG_EVENT_LOOP_OWNER` toggle were retired in [#3164](https://github.com/jwbron/egg/issues/3164), so this is the only mode). Agent pods are short-lived one-shot Jobs rather than long-running containers. Between dispatched events a role has no running pod — and that is normal. The health monitor and heartbeat coordinator are both placed in **orchestrator mode** by the `ConcurrentPhaseExecutor` at event-loop setup: - **Health monitor (`set_orchestrator_mode(True)`):** Roles with no active one-shot Job are treated as legitimately idle. Heartbeat-timeout, progress-stall, and container-exit tripwires fire only while that role's Job is active. A pod that exits silently **mid-event** (Job active but no heartbeat) still alerts normally. The active-Job set is refreshed every poll tick by the event loop via `set_active_roles(roles)`, so coverage is current. - **Heartbeat coordinator (`set_orchestrator_mode(True)`):** The gateway-session refresh side-effect (`should_fan_out_gateway_session`) is suppressed. Session refresh happens at spawn time (slice-4 worktree re-attach), not via heartbeat fan-out. diff --git a/docs/index.md b/docs/index.md index 371bf024d7..303ce626d8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -83,7 +83,7 @@ This index helps both humans and LLMs navigate the documentation efficiently. | [SDLC Contract](reference/sdlc-contract.md) | Full `egg-contract` command reference for tracking tasks, commits, decisions | | [MCP Deployment Tools](reference/mcp-deployment-tools.md) | Six k8s-facing MCP tools: `get_deployment_context`, `validate_deployment_manifests`, `prune_stale_worktrees`, `validate_network_isolation`, `rebuild_and_rollout`, `get_service_logs` | | [Agent MCP Tools](reference/agent-tools.md) | In-process SDK MCP tools sandbox agents call on the `tool_use` stream (45 verbs across 7 namespaces: `mcp__sdlc__*`, `mcp__brc__*`, `mcp__phase__*`, `mcp__progress__*`, `mcp__task__*`, `mcp__confluence__*`, `mcp__jira__*`); on by default — set `EGG_MCP_TOOLS=false` to opt out | -| [Agent Wait Patterns](reference/agent-wait-patterns.md) | §0 the agent contract (agents never wait on the bus — the consensus wrapper owns all blocking waits and invokes agents one-shot per event; #2908/#3157), the wrapper's `egg-orch message wait-loop` idiom and per-state filters, the five anti-patterns the event pump retired, the `egg-orch message wait` exit-code contract, the `HEARTBEAT` metadata schema, the `EGG_MESSAGE_POLL_MAX_WAIT` / `EGG_ORCH_WAITRESS_THREADS` env-var couplings, §7 host-side `egg-orch pipeline wait-status` for event-driven pipeline monitoring, §10 BRC event-pump wrapper wait surface (conditional filter construction, heartbeat ownership migration, `slice_id` propagation invariant, and idle-budget escalation path), and §10.9 per-event prompt composer + preamble collapse (prompt shape, full git-log delta, `EGG_BRC_MEMORY` matrix) | +| [Agent Wait Patterns](reference/agent-wait-patterns.md) | §0 the agent contract (agents never wait on the bus — the orchestrator owns all waiting and spawns agents one-shot per event; #2908/#3157/#3164), the wrapper's `egg-orch message wait-loop` idiom and per-state filters, the five anti-patterns the event pump retired, the `egg-orch message wait` exit-code contract, the `HEARTBEAT` metadata schema, the `EGG_MESSAGE_POLL_MAX_WAIT` / `EGG_ORCH_WAITRESS_THREADS` env-var couplings, §7 host-side `egg-orch pipeline wait-status` for event-driven pipeline monitoring, §10 BRC event-pump wrapper wait surface (conditional filter construction, heartbeat ownership migration, `slice_id` propagation invariant, and idle-budget escalation path), and §10.9 per-event prompt composer + preamble collapse (prompt shape, full git-log delta, `EGG_BRC_MEMORY` matrix) | | [Jira Wrapper](reference/jira-wrapper.md) | `/api/v1/jira/*` gateway endpoints — read verbs (ticket read, JQL search with static project-scope extraction, ticket comments, GET-only execute passthrough) plus the bounded write extension (`ticket/create`, `ticket/edit`, `ticket/comment/add`, `issue-link/create`); private-mode only; project allowlist via `config/context-filters.yaml`; per-verb body schema with size caps; ADF wrapping for plain-text `description` / `comment` bodies (`gateway/jira_adf.py`); caller-supplied idempotency key with 5-minute in-process cache (`gateway/jira_idempotency.py`); operator-configurable `jira.link_types` / `jira.epic_link_field` knobs; audit redaction never logs body content; `not_found` envelope on read 404s | | [Confluence Wrapper](reference/confluence-wrapper.md) | `/api/v1/confluence/*` read-only gateway endpoints (page read, descendants, footer/inline comments with v1 fallback, space list/pages, CQL search with static space-scope extraction, GET-only execute passthrough); private-mode only; space allowlist via `config/context-filters.yaml`; `not_found` envelope; response redaction (`accountId` / `emailAddress` / user-profile `_links.webui` / user-profile `_links.self`); future write-verb extension points | | [Conditional ACK](reference/conditional-ack.md) | Reviewer verdict variant: ACK + `--pre-merge-condition "..."` attaches a merge-time human obligation (e.g. `git mv`) that surfaces in `egg-orch consensus status` and in a "Pre-merge Obligations" section on the auto-created PR body | diff --git a/docs/reference/agent-tools.md b/docs/reference/agent-tools.md index 315158e22a..9071dc0450 100644 --- a/docs/reference/agent-tools.md +++ b/docs/reference/agent-tools.md @@ -127,9 +127,9 @@ that requires the handler docstring to explain why no CLI exists. | `mcp__brc__confirm` | Signal CONFIRMED — producer acknowledges all reviewer ACKs. Returns `ok=True` when the transition to CONFIRMED succeeded. Returns `ok=False` with `status="pending_acks"` when the orchestrator rejected the transition (e.g. not yet fully ACKed, stale ACKs); this is transient — retry after polling for outstanding ACKs. Equivalent to CLI exit code 0 vs 2. | `handlers.brc.brc_confirm` | `egg-orch consensus confirmed` | | `mcp__brc__get_state` | Full structured consensus state (JSON; accepts `verbose: bool`). Slice-aware (#2761): scopes to the per-slice BRC tracker via `slice_id` (defaults to `EGG_SLICE_ID`), so a per-slice agent sees its own slice's consensus rather than a pipeline-level reconstruction. | `handlers.brc.brc_get_state` | `egg-orch brc get-state` *(verb-level CLI alias added in #2908; `cli_command=None` in MCP registry — schema is in `schemas.py`, not argparse)* | | `mcp__brc__list_blocking` | Return the list of agent roles currently blocking consensus (derived view). | `handlers.brc.brc_list_blocking` | `egg-orch brc list-blocking` *(verb-level CLI alias added in #2908; `cli_command=None` in MCP registry)* | -| `mcp__brc__send_heartbeat` | Emit a structured `HEARTBEAT` (schema-validated, per-role deduped, rate-limited) to the dedicated `/heartbeat` endpoint on a state transition. Valid states: `WORKING`, `WAITING_ON_ROLE` (+ `waiting_on=`), `WAITING_FOR_EVENT`, `PROPOSED`, `IDLE` — but agents never emit `WAITING_FOR_EVENT`: the consensus wrapper owns bus waits and the heartbeats emitted while blocked (#2908). | `handlers.message.message_heartbeat` | `egg-orch message heartbeat` | +| `mcp__brc__send_heartbeat` | Emit a structured `HEARTBEAT` (schema-validated, per-role deduped, rate-limited) to the dedicated `/heartbeat` endpoint on a state transition. Valid states: `WORKING`, `WAITING_ON_ROLE` (+ `waiting_on=`), `WAITING_FOR_EVENT`, `PROPOSED`, `IDLE` — but agents never emit `WAITING_FOR_EVENT`: the orchestrator owns all waiting and spawns agents one-shot per event (#2908/#3164). | `handlers.message.message_heartbeat` | `egg-orch message heartbeat` | -> **No blocking-wait tools, by design** (#2211, #2908, #3157). The MCP wait tools were removed in #2211 because long polls don't fit the MCP transport — both transports cap tool calls below typical quiet-phase intervals (~30 s streamable-HTTP, ~60 s in-process SDK), and every cap-elapsed return is a wasted LLM turn. They were not replaced with an agent-tier Bash idiom: blocking waits belong to other tiers entirely. Sandbox-side, the consensus wrapper blocks in `egg-orch message wait` / `wait-loop` and invokes the agent one-shot per event — agents never wait on the bus (`docs/reference/agent-wait-patterns.md` §0). Host-side, the SDLC skill blocks in `egg-orch pipeline wait-status` via Bash. +> **No blocking-wait tools, by design** (#2211, #2908, #3157). The MCP wait tools were removed in #2211 because long polls don't fit the MCP transport — both transports cap tool calls below typical quiet-phase intervals (~30 s streamable-HTTP, ~60 s in-process SDK), and every cap-elapsed return is a wasted LLM turn. They were not replaced with an agent-tier Bash idiom: blocking waits belong to other tiers entirely. Sandbox-side, the orchestrator owns all waiting — it derives the next BRC event in-process and spawns the agent one-shot per event (the in-pod wait arm was retired by #3164); agents never wait on the bus (`docs/reference/agent-wait-patterns.md` §0). Host-side, the SDLC skill blocks in `egg-orch pipeline wait-status` via Bash. | `mcp__brc__read_peer_artifact` | Read the BRC transcript for a phase from TWO merged sources (#3076 / #3077 phase 1): the orchestrator's live `/brc-transcript` route (the message store holds exactly the in-flight phase — a peer's `CONSENSUS_PROPOSE` is visible the moment it is sent, Delphi-redacted server-side for unreviewed reviewers) and the local `.egg-state/brc-history/-.json` files (phases completed before spawn; per-slice partition `-implement-.json` when `EGG_SLICE_ID` is set and `phase == "implement"`, with the `unattributed` sibling merged in unless `include_unattributed=False`). Records dedup by message id, sorted by timestamp. Optional filters: `peer_role` / `producer_role` (alias), `message_type` (str or list). `limit` / `cursor` pagination (default `limit=50`, max 500). The identifier / pipeline id are resolved server-side from `EGG_ISSUE_NUMBER` / `EGG_PIPELINE_ID` (agents cannot pass an arbitrary id; path-traversal hardening). Returns `{items: [...], next_cursor: , total_available: , skipped_malformed: , live: , hint?: }` — `live` says whether the live route contributed; `hint` is present only when both sources are empty: with the live route reachable that genuinely means no BRC messages for the phase yet, without it the emptiness is structural (#3076; brc-history is written at phase *completion*) and the hint points at the event-payload fallback (`pending_reviews[].proposal_commit_sha` + `egg-artifact get --ref ` for spec-registered artifacts, or `git log --not origin/ -p` for code files). | `handlers.brc.brc_read_peer_artifact` | `egg-orch brc read-peer-artifact` *(slice-5 of #2908; thin wrapper, registration still `cli_command=None` — see callout below)* | | `mcp__brc__resolve_obligation` | Mark a reviewer's conditional-ACK obligation as satisfied in-cycle (#2338). Required: `reviewer_role`, `producer_role`. Optional: `commit_sha`, `note`. The matrix keeps the obligation text for audit, but `get_pre_merge_conditions` filters resolved entries — the PR body and HITL gate stop surfacing the obligation. The orchestrator persists a `CONSENSUS_OBLIGATION_RESOLVED` message so the resolution survives orchestrator restart, and rejects `resolver_role == producer_role` so a producer cannot self-resolve their own obligation. Resolution is per-version: any later ACK / NACK / invalidate on the same edge resets the resolved flag. | `handlers.brc.brc_resolve_obligation` | `egg-orch brc resolve-obligation` *(slice-5 of #2908; thin wrapper, registration still `cli_command=None` — see callout below)* | diff --git a/docs/reference/agent-wait-patterns.md b/docs/reference/agent-wait-patterns.md index b51010afd6..45929e1eec 100644 --- a/docs/reference/agent-wait-patterns.md +++ b/docs/reference/agent-wait-patterns.md @@ -20,21 +20,22 @@ links here rather than duplicating the contract. ## 0. The Agent Contract: Agents Never Wait on the Bus -**Agents do not wait. The wrapper does.** Since #2908 the in-pod consensus -wrapper (`orchestrator/consensus_wrapper.py`) is a deterministic event -pump: it polls `egg-orch brc next-action`, blocks in `egg-orch message -wait-loop` between actionable events, and invokes the agent **one-shot per -event**. The agent handles the single event in its prompt (propose, ACK, -NACK, or confirm) and exits naturally; the wrapper owns every blocking -read on the bus and the heartbeats emitted while blocked. The -authoritative agent-facing wording is the per-event prompt composed in -`orchestrator/routes/event_prompt.py`: +**Agents do not wait. The orchestrator does.** Since #2908 the BRC loop +has been deterministic; [#3164](https://github.com/jwbron/egg/issues/3164) +retired the in-pod wait arm and moved ownership to the orchestrator. The +orchestrator (`orchestrator/event_loop.py`) derives `egg-orch brc +next-action` in-process and spawns a **one-shot pod per actionable event**. +The agent handles the single event in its prompt (propose, ACK, NACK, or +confirm) and exits naturally; the orchestrator owns all waiting — there is +no in-pod blocking wait and no in-pod background heartbeat (both retired in +#3164). The authoritative agent-facing wording is the per-event prompt +composed in `orchestrator/routes/event_prompt.py`: > "When you have acted (proposed, ACKed, NACKed, or confirmed), exit -> naturally — the wrapper polls `egg-orch brc next-action` and re-invokes -> you with the next actionable event. Do NOT block on `egg-orch message -> wait-loop` yourself: the wrapper owns the wait and the heartbeat -> (#2908 slice-2)." +> naturally — the orchestrator derives `egg-orch brc next-action` +> in-process and re-spawns you one-shot with the next actionable event. +> Do NOT block on `egg-orch message wait-loop` yourself: the orchestrator +> owns the wait and spawns you one-shot per event (#3164)." Consequences for anyone writing or revising prompts, role contracts, or agent rules: @@ -47,13 +48,19 @@ agent rules: the one-shot model keeps turns bounded, with the [BRC memory file](../architecture/brc-memory.md) as the cross-turn state carrier. -- Everything below this section documents the **wrapper-tier** mechanics. - It is reference material for maintainers of the wrapper and the - orchestrator wait routes, not instructions for agents. - -[#3064](https://github.com/jwbron/egg/issues/3064) lifts the event loop -out of the pod entirely (orchestrator-driven on-demand spawning); the -invariant — agents never block on the bus — holds at every tier. +- Everything below this section documents the **wait-tier** mechanics. + Post-#3164 the orchestrator owns the loop (`orchestrator/event_loop.py`) + and the in-pod wait-loop + 30 s background heartbeat are retired; the + detailed `egg-orch message wait-loop` idiom below is retained as + reference material for the wait surface itself (still driven server-side + for the overseer/tooling and the historical in-pod path), not as + instructions for agents. + +[#3064](https://github.com/jwbron/egg/issues/3064) lifted the event loop +out of the pod (orchestrator-driven on-demand spawning) and +[#3164](https://github.com/jwbron/egg/issues/3164) retired the in-pod wait +arm so orchestrator ownership is the only mode; the invariant — agents +never block on the bus — holds at every tier. ## 1. The Canonical Idiom @@ -1338,19 +1345,23 @@ the configured thread count, raise it. > the architecture doc covers the *why* and the cross-slice rollout > plan, this section covers the *wait* surface contract. > -> The event-pump model is now the only path: slice-4 deleted the +> The event-pump model is the only path: slice-4 deleted the > legacy capped-restart wrapper template, the `_RECOVERY_SYSTEM_PROMPT`, > the SSE `consensus.reached` machinery, and the `MAX_CONSENSUS_RESTARTS` > cap, and removed the agent-side heartbeat + gateway-session keep-alive -> from `sandbox/egg_agent_tools/handlers/message.py`. The wrapper holds -> the BRC wait, dispatches the agent one-shot per actionable event, and -> emits heartbeats / refreshes the gateway session from background -> subshells inside the wrapper bash. The §1–§9 contracts above still -> apply at the `egg-orch message wait-loop` call site itself; what -> changed is *who calls it* (the wrapper, not the agent's -> `message_wait_loop`). See [Rollback plan](../architecture/orchestrator.md#rollback-plan) -> for the `git revert` regression path if production traffic ever -> needs to fall back to the legacy capped-restart model. +> from `sandbox/egg_agent_tools/handlers/message.py`. +> **[#3164](https://github.com/jwbron/egg/issues/3164) then retired the +> in-pod wait arm entirely:** the orchestrator (`orchestrator/event_loop.py`) +> now owns the BRC wait — it derives the next action in-process and spawns +> a one-shot pod per actionable event. There is no longer an in-pod +> `egg-orch message wait-loop` between events and no 30 s background +> heartbeat subshell. The §1–§9 contracts below still describe the +> `egg-orch message wait-loop` surface itself (still server-side, still +> used by the overseer/tooling, and the shape the orchestrator's in-process +> derivation mirrors); §10.1–§10.9 below describe the **historical** in-pod +> wrapper mechanics, kept for lineage. #3164 retired the in-pod arm and the +> `EGG_EVENT_LOOP_OWNER` flag with no rollback flag — orchestrator +> ownership is the only mode. ### 10.1 The shape in one diagram @@ -1375,13 +1386,15 @@ STEADY STATE (event-pump, the only path after slice-4): INVOKE: python3 -m egg_agent ``` -The wait still happens — it just moves out of the agent and into the -wrapper bash. The §1 idiom is what the wrapper runs; the agent no -longer runs it directly. The model is invoked **once per actionable -event**, so the seam where Claude / qwen3.7-max / future-LLM could -decline to re-enter the wait (the #2906 fall-out) is gone — only the -deterministic bash loop decides when to wait, when to invoke, and -when to confirm. +The wait still happens — historically (pre-#3164) it moved out of the +agent and into the wrapper bash; post-#3164 it moved further out, into +the orchestrator's in-process event loop (`orchestrator/event_loop.py`), +which derives the next action and spawns a one-shot pod per actionable +event rather than running an in-pod `wait-loop`. Either way the model is +invoked **once per actionable event**, so the seam where Claude / +qwen3.7-max / future-LLM could decline to re-enter the wait (the #2906 +fall-out) is gone — only a deterministic loop decides when to wait, when +to invoke, and when to confirm. ### 10.2 Wait-filter construction is conditional on `is_role_confirmed` @@ -1405,19 +1418,22 @@ HTTP 400 rejection at `/messages/wait` (see [#2482](https://github.com/jwbron/egg/issues/2482)) cannot land here silently. -### 10.3 Heartbeat ownership lives in the wrapper (#2036 migration completed in slice-4) +### 10.3 Heartbeat ownership (#2036 migration completed in slice-4; in-pod heartbeat retired by #3164) -The wrapper owns BRC heartbeating: a background subshell fires -`egg-orch message heartbeat` every 30 s while the wrapper's own -`egg-orch message wait-loop` call is blocking, in parallel with the -wait. The pre-#2908 agent-side path — `message_wait_loop` in +**Retired by [#3164](https://github.com/jwbron/egg/issues/3164).** Historically +(post-#2036, pre-#3164) the in-pod wrapper owned BRC heartbeating: a +background subshell fired `egg-orch message heartbeat` every 30 s while the +wrapper's own `egg-orch message wait-loop` call was blocking, in parallel +with the wait. The pre-#2908 agent-side path — `message_wait_loop` in `sandbox/egg_agent_tools/handlers/message.py` self-emitting `WAITING_FOR_EVENT` once on entry plus every 60 s while blocked (see §4 — "the wait primitive owns its lifecycle") — was **deleted in slice-4 task-4-2** alongside the legacy capped-restart wrapper -template. The agent is now one-shot per actionable event, so there -is no in-pod loop left to emit heartbeats between events; the -wrapper is the only process alive across the full BRC cycle. +template. #3164 then removed the in-pod wait arm and the 30 s background +heartbeat subshell entirely: with the orchestrator spawning a one-shot pod +per actionable event, there is no long-lived in-pod process to emit +heartbeats between events. The gateway session is refreshed at spawn time +(see §10.4), not via a between-events heartbeat fan-out. The schema in §4 is unchanged across the #2036 migration; only the *emitter* moved. The @@ -1685,13 +1701,14 @@ production pipelines run the adversarial re-review path; operators that need to fall back to the slice-1 inert-reader behaviour can still set `EGG_BRC_MEMORY=write-only` explicitly. -#### 10.9.5 The server-side BRC preamble is collapsed; the wrapper owns the lifecycle now +#### 10.9.5 The server-side BRC preamble is collapsed; the orchestrator owns the lifecycle now `_build_brc_preamble` at `orchestrator/routes/pipelines.py:12180` (or look up by function name — the line drifts as the file edits) is **collapsed** in slice-3. The agent no longer has to be re-taught -the lifecycle on every spawn — the wrapper bash now drives the -sequencing deterministically (§10.1) and constructs the +the lifecycle on every spawn — the orchestrator now drives the +sequencing deterministically (post-#3164 it derives the next action +in-process and spawns a one-shot pod per event) and constructs the filter set conditionally (§10.2), so the preamble pieces that used to teach `egg-orch message wait-loop`, cursor threading, the STAY-ALIVE post-confirm loop, and the pre-confirm `CONSENSUS_CONFIRMED` @@ -1699,7 +1716,7 @@ exclusion are all removed from the prompt the agent sees. | Removed | Why | |---------|-----| -| Producer Lifecycle step 4 wait-loop plumbing | Wrapper holds the wait (§10.1). | +| Producer Lifecycle step 4 wait-loop plumbing | The orchestrator owns the wait (post-#3164; §10.1). | | Producer Lifecycle step 6 STAY-ALIVE loop | Wrapper loops to the next `brc next-action` instead of the agent re-entering a wait. | | Cursor / `--since` threading guidance | Cursor threading is automatic and wrapper-internal under the event-pump path. | | Pre-confirm-wait foot-gun guidance (anti-pattern 5, §10.2) | Wrapper's conditional filter prevents the regression at the emission site. | diff --git a/docs/reference/orchestrator-cli.md b/docs/reference/orchestrator-cli.md index bc45daa4c4..878a3ae08a 100644 --- a/docs/reference/orchestrator-cli.md +++ b/docs/reference/orchestrator-cli.md @@ -46,9 +46,9 @@ Run `egg-orch --help` for full usage. All commands support `--json` for machine- | `egg-orch overseer file-issue [] --anomaly-type --priority --agent-role --anomaly-signature --issue-title-file --issue-body-file [--parent-alert-message-id ] [--dry-run] [--json]` | File a GitHub issue from the overseer role (advisor-gated). Checks `filed-issues.jsonl` + `gh issue list` for deduplication; skips filing if a matching open issue exists. Default stdout is plain text (`Filed issue #N (...)` or `Existing issue #N already covers ...`); with `--json`, prints JSON `{"issue_number": int, "filed": bool, "dedup_match": int\|null}`; with `--dry-run` (no `gh` invocation either way): if a dedup match is found, prints the same `--json` shape (with `issue_number`/`dedup_match` populated); otherwise prints `{"issue_number": null, "filed": false, "dedup_match": null, "dry_run": true, "argv": [...], "title": str, "body_bytes": int}`. Requires `EGG_PIPELINE_REPO` env var. | | `egg-orch overseer consult-advisor [] --inputs-file [--output-file ] [--recent-log-bytes-cap ] [--json]` | Consult the advisor for a structured `AdvisorVerdict` (sandbox-side LLM call). Reads a JSON inputs file with `classification`, `health_alerts`, `progress_events`, and `recent_log_lines`. Returns `decision` (`alert`\|`file_issue`\|`watch`), `priority`, `alert_summary`, `alert_detail`, `issue_title`, `issue_body`, and `reasoning`. Without `--output-file`, the JSON verdict is written to stdout; with `--output-file`, the verdict is written to that path and stdout shows a confirmation message (pass `--json` to additionally echo the verdict to stdout). When `[]` (or `EGG_PIPELINE_ID`) is set, the verb reads `PipelineConfig.overseer_advisor_model` and `PipelineConfig.overseer_advisor_recent_log_bytes_cap` from the orchestrator status endpoint and passes the configured values to `consult_advisor`; falls back to the `opus` model default and 256 KiB byte cap if the pipeline ID is absent or the lookup fails. `--recent-log-bytes-cap` overrides the `PipelineConfig.overseer_advisor_recent_log_bytes_cap` value; oldest lines are dropped first when the block exceeds the cap. `0` disables the cap. | | `egg-orch message poll [] [--since ] [--limit ]` | Poll for messages from other agents (concurrent mode) | -| `egg-orch message wait [] --for ... [--timeout N] [--since ] [--from-producer ]... [--slice ]` | **Wrapper-internal** (#2908/#3157; agents never wait on the bus) — block until a **new** typed BRC event arrives (cursor-less calls start at stream tip — already-seen events are skipped). Cursor threading across re-entries is automatic when `EGG_AGENT_ROLE` is set (issue #2323). Pass `--since ` to resume from a specific anchor when needed. `--from-producer` (repeatable; also accepts comma-separated values, e.g. `--from-producer coder,tester`) restricts wakes to messages from the named senders; defaults to `$EGG_WAIT_PRODUCER_ALLOWLIST`. `--slice` restricts wakes to messages for the named slice (messages with null `slice_id` always pass through); defaults to `$EGG_SLICE_ID`. Exit 0 = matched, 1 = timeout, 2 = transient (retry-safe), 3 = permanent. See [Agent Wait Patterns §3](agent-wait-patterns.md#3-exit-code-contract-for-egg-orch-message-wait) | -| `egg-orch message wait-loop [] --for ... [--since ] [--from-producer ]... [--slice ]` | **Wrapper-internal** (#2908/#3157): the consensus wrapper's canonical blocking wait — loops `message wait` server-side until a new matching event arrives (defaults to stream-tip; cursor threading across re-entries is automatic when `EGG_AGENT_ROLE` is set — issue #2323). `--from-producer` and `--slice` auto-apply from `$EGG_WAIT_PRODUCER_ALLOWLIST` / `$EGG_SLICE_ID` when set by the spawner (#2725). Agents must never call this — the wrapper owns the wait and invokes the agent one-shot per event. Do not wrap in an outer shell loop. See [Agent Wait Patterns §1](agent-wait-patterns.md#1-the-canonical-idiom) | -| `egg-orch message heartbeat [] --state [--waiting-on ] [--since ]` | Emit a structured `HEARTBEAT` message on state transitions. `WAITING_ON_ROLE` requires `--waiting-on`. `WAITING_FOR_EVENT` is emitted automatically by `egg-orch message wait-loop` (a wrapper-internal call) while blocked — agents never emit it. Rate-limited by `EGG_HEARTBEAT_RATE_LIMIT` (per-role, 429 on exceed). See [Agent Wait Patterns §4](agent-wait-patterns.md#4-heartbeat-message-type) | +| `egg-orch message wait [] --for ... [--timeout N] [--since ] [--from-producer ]... [--slice ]` | **Not for agents** (#2908/#3157/#3164; agents never wait on the bus — the orchestrator owns all waiting and derives the next BRC event in-process) — block until a **new** typed BRC event arrives (cursor-less calls start at stream tip — already-seen events are skipped). Cursor threading across re-entries is automatic when `EGG_AGENT_ROLE` is set (issue #2323). Pass `--since ` to resume from a specific anchor when needed. `--from-producer` (repeatable; also accepts comma-separated values, e.g. `--from-producer coder,tester`) restricts wakes to messages from the named senders; defaults to `$EGG_WAIT_PRODUCER_ALLOWLIST`. `--slice` restricts wakes to messages for the named slice (messages with null `slice_id` always pass through); defaults to `$EGG_SLICE_ID`. Exit 0 = matched, 1 = timeout, 2 = transient (retry-safe), 3 = permanent. See [Agent Wait Patterns §3](agent-wait-patterns.md#3-exit-code-contract-for-egg-orch-message-wait) | +| `egg-orch message wait-loop [] --for ... [--since ] [--from-producer ]... [--slice ]` | **Not for agents** (#2908/#3157/#3164): the canonical blocking wait — loops `message wait` server-side until a new matching event arrives (defaults to stream-tip; cursor threading across re-entries is automatic when `EGG_AGENT_ROLE` is set — issue #2323). Post-#3164 the consensus wrapper no longer calls this between BRC events — the orchestrator derives the next action in-process (`egg-orch brc next-action`) and spawns the agent one-shot per event. The CLI surface still exists (overseer/tooling). `--from-producer` and `--slice` auto-apply from `$EGG_WAIT_PRODUCER_ALLOWLIST` / `$EGG_SLICE_ID` when set by the spawner (#2725). Agents must never call this — the orchestrator owns the wait and spawns the agent one-shot per event. Do not wrap in an outer shell loop. See [Agent Wait Patterns §1](agent-wait-patterns.md#1-the-canonical-idiom) | +| `egg-orch message heartbeat [] --state [--waiting-on ] [--since ]` | Emit a structured `HEARTBEAT` message on state transitions. `WAITING_ON_ROLE` requires `--waiting-on`. `WAITING_FOR_EVENT` is emitted automatically by `egg-orch message wait-loop` (not an agent-facing call; post-#3164 the orchestrator owns all waiting) while blocked — agents never emit it. Rate-limited by `EGG_HEARTBEAT_RATE_LIMIT` (per-role, 429 on exceed). See [Agent Wait Patterns §4](agent-wait-patterns.md#4-heartbeat-message-type) | | `egg-orch message status []` | Get message bus status (concurrent mode) | | `egg-orch signal readiness [] --state [--reason "..."]` | Signal readiness state (concurrent mode) | | `egg-orch push` | Push current branch via the gateway. The gateway rejects pushes that modify restricted paths (`403 restricted_path_modified`); drop the offending edits and re-propose with `--pre-merge-condition` ([#2039](https://github.com/jwbron/egg/issues/2039)) | @@ -79,7 +79,7 @@ Agent role can be omitted when `EGG_AGENT_ROLE` is set. | `GATEWAY_URL` | Gateway URL (default: `http://egg-gateway:9848`) | | `EGG_CONCURRENT_MODE` | `true` when running in concurrent execution mode | | `EGG_BRC_MEMORY` | BRC memory writer mode for reviewers. `full` (default since slice-4) — handlers write and the event-pump reads the file on re-entry; `write-only` — handlers populate the per-role memory file but the reader stays inert; `off` — writes are no-ops (one-release rollback escape hatch). See [BRC Memory Artifact](../architecture/brc-memory.md). | -| `EGG_EVENT_LOOP_OWNER` | BRC event-loop ownership mode (#3064). `pod` (default) — today's behavior: each agent container runs the in-pod event-pump wait-loop in `consensus_wrapper.py`. `orchestrator` — the orchestrator owns the loop and spawns a one-shot pod per actionable BRC event (slice-2+). Values are matched case-insensitively (e.g. `Orchestrator`, `POD` are accepted). Unset/empty defaults to `pod`; any unrecognised value raises `ValueError` at read time (no silent fallback — see the #3023 post-mortem). The one-shot arm in the wrapper is dormant until `orchestrator` mode is selected AND the slice-2 spawner exists. | +| `EGG_EVENT_LOOP_OWNER` | **Removed in [#3164](https://github.com/jwbron/egg/issues/3164).** During the #3064 rollout this flag selected `pod` (in-pod event-pump wait-loop in `consensus_wrapper.py`) vs `orchestrator` (orchestrator-owned event loop spawning one-shot pods). #3164 retired the in-pod wait arm — orchestrator ownership is the only mode — and removed the read from `orchestrator/env_config.py`. The variable is no longer consulted; operators can drop the row. | | `EGG_MESSAGE_POLL_INTERVAL` | Suggested message polling interval in seconds (default: 30) | | `EGG_MESSAGE_POLL_MAX_WAIT` | Server-side cap (seconds) on `message wait --timeout`. Default `60`, minimum `1`. Values `> 90` trigger a startup `warnings.warn` + WARNING log because the gateway's baked-in Squid `read_timeout` / `request_timeout` directives cap backend long-polls at ~60s — raising the cap above that requires a gateway image rebuild, not a ConfigMap edit. See [Agent Wait Patterns §6](agent-wait-patterns.md#6-egg_message_poll_max_wait--long-poll-cap-coupling). | | `EGG_SLICE_ID` | Set by `kubernetes_spawner` for every slice-scoped agent. When set, `message wait` / `message wait-loop` only match messages whose `metadata.slice_id` equals this value OR is null (pipeline-level passthrough — OVERSEER_ALERT and global phase signals continue to wake every waiter). Override with `--slice`. See [Agent Wait Patterns — Auto-scoping](agent-wait-patterns.md#auto-scoping-by-slice-and-producer-allowlist-2725). | diff --git a/orchestrator/concurrent_executor.py b/orchestrator/concurrent_executor.py index 2f59d7063d..bdcbbc3ec0 100644 --- a/orchestrator/concurrent_executor.py +++ b/orchestrator/concurrent_executor.py @@ -149,7 +149,10 @@ def spawn_event( ) -> Any: agent_role = self._agent_role(role) branch = self._ex.get_worktree_branch(agent_role, slice_id=self._slice_id) - env = {**self._ex.get_agent_env(agent_role), "EGG_EVENT_LOOP_OWNER": "orchestrator"} + # The one-shot wrapper keys on EGG_EVENT_ACTION (injected by + # ``spawn_event_job`` via ``event_action`` below), not on an + # ownership flag — the EGG_EVENT_LOOP_OWNER env was retired in #3164. + env = self._ex.get_agent_env(agent_role) command, upstream, upstream_model = self._ex._build_event_spawn_params(agent_role) return self._ex.spawn_fn( role=agent_role, @@ -422,38 +425,15 @@ def spawn_all( for role in roles: tracker.register_agent(role.value) - # #3064 slice-2: in orchestrator-ownership mode the orchestrator owns - # the BRC event loop and spawns a one-shot pod per actionable event - # — NO agents are spawned up front. The tracker is still registered - # above (the event loop derives against it). With the flag unset / - # ``pod`` (the default) this branch is skipped and behavior is - # byte-identical to before. - if self._event_loop_owner() == "orchestrator": - self._start_event_loop(roles, tracker) - return [] - - # A producer with no work in this slice (e.g. a documenter on a - # code-only slice) is no longer pre-seeded here (#3027 retired the - # #2581 pre-seed). Instead it stays spawned and submits a generic - # no-op propose (``no_changes_needed=true``) at runtime, which the - # consensus protocol accepts as a non-blocking, durable no-op — - # robust to restart / reconstruction in a way the in-memory seed - # never was. - return self._spawn_roles(roles, agent_prompts or {}) - - @staticmethod - def _event_loop_owner() -> str: - """Return the BRC event-loop ownership mode (``pod`` | ``orchestrator``). - - Lazy dual-path import (repo root vs ``orchestrator/`` on sys.path), - mirroring ``consensus_wrapper._event_loop_owner``. An invalid value - raises loudly (the #3023 no-silent-fallback contract). - """ - try: - from orchestrator.env_config import get_event_loop_owner - except ImportError: - from env_config import get_event_loop_owner # type: ignore[no-redef] - return get_event_loop_owner() + # #3164: the orchestrator unconditionally owns the BRC event loop + # and spawns a one-shot pod per actionable event — NO agents are + # spawned up front. The tracker is registered above (the event loop + # derives against it). ``agent_prompts`` is accepted for caller + # compatibility but unused: the wrapper composes its own per-event + # prompt via ``compose_event_prompt``. + del agent_prompts + self._start_event_loop(roles, tracker) + return [] def _start_event_loop(self, roles: list[AgentRole], tracker: Any) -> OrchestratorEventLoop: """Construct and start the orchestrator-owned event loop (#3064 slice-2). @@ -519,13 +499,6 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None: roles=[r.value for r in roles], ) - # #3064 slice-5: set orchestrator mode on the health monitor and - # heartbeat coordinator so their tripwire/refresh behavior reflects - # the ownership mode (roles with no active Job are normal in - # orchestrator mode; gateway-session refresh via heartbeat fan-out - # is suppressed). - self._enable_orchestrator_mode_surfaces() - return loop def _publish_active_roles(self, roles: set[str]) -> None: @@ -550,35 +523,6 @@ def _publish_active_roles(self, roles: set[str]) -> None: except Exception: # noqa: BLE001 — best-effort pass - def _enable_orchestrator_mode_surfaces(self) -> None: - """Propagate orchestrator mode to downstream surfaces. - - In orchestrator mode: - - The health monitor suppresses alerts for roles with no active Job. - - The heartbeat coordinator suppresses gateway-session fan-out - (refresh happens at spawn time from slice-4 worktree re-attach). - - Absent-sender heartbeats between events trip nothing. - - Best-effort: a missing health monitor or coordinator is tolerated - (unit tests often stand up only the component under test). - """ - try: - from health_monitor import get_health_monitor - - hm = get_health_monitor() - if hm is not None: - hm.set_orchestrator_mode(True) - except Exception: # noqa: BLE001 — best-effort - pass - - try: - from heartbeat import get_heartbeat_coordinator - - hc = get_heartbeat_coordinator() - hc.set_orchestrator_mode(True) - except Exception: # noqa: BLE001 — best-effort - pass - def owns_event_loop(self) -> bool: """True when the orchestrator-owned BRC event loop drives this phase. @@ -590,8 +534,7 @@ def owns_event_loop(self) -> bool: incomplete-consensus HITL escalation (``return 1``) on this method so an orchestrator-owned slice that never converged is treated as the failure it is, rather than falling through to a bare ``return 0``. Set - only when ``spawn_all`` took the orchestrator-ownership branch and - started the loop. + once ``spawn_all`` started the loop (always, post-#3164). """ return self._event_loop is not None @@ -600,8 +543,9 @@ def stop_event_loop(self) -> None: Called by the completion-poll site on every exit path so the daemon thread does not outlive the phase (and stops requesting one-shot - spawns) once consensus is reached, times out, or fails. A no-op in - pod mode, where no loop was started. + spawns) once consensus is reached, times out, or fails. A no-op if + the loop was never started (e.g. a phase that short-circuited before + ``spawn_all``). """ loop = self._event_loop if loop is None: diff --git a/orchestrator/consensus_wrapper.py b/orchestrator/consensus_wrapper.py index c742f3d55e..cdd126a523 100644 --- a/orchestrator/consensus_wrapper.py +++ b/orchestrator/consensus_wrapper.py @@ -1,75 +1,35 @@ """Build consensus-wrapped commands for concurrent agent containers. -EVENT-PUMP MODEL (#2908) -~~~~~~~~~~~~~~~~~~~~~~~~ -The consensus wrapper invokes the agent one-shot per actionable BRC -event. A deterministic bash loop drives the lifecycle: - -* fetch BRC state via ``egg-orch brc get-state`` (slice-1 task-1-3); -* ask ``egg-orch brc next-action`` what to do (slice-1 task-1-1); -* on ``wait`` block on ``egg-orch message wait-loop`` while emitting - ``egg-orch message heartbeat`` (heartbeat lineage #2036 + #2451 — - the heartbeat carries ``slice_id``, so ``_maybe_attach_slice_id`` in - the orchestrator fan-out refreshes the slice-scoped container session - as a side effect of every wrapper heartbeat); -* on ``propose|ack|nack`` invoke the agent one-shot via - ``python3 -m egg_agent`` with the per-event prompt composed by - ``orchestrator/routes/event_prompt.py:compose_event_prompt`` - (slice-3 task-3-1); -* on ``confirm`` / ``complete`` call ``egg-orch consensus confirmed``; -* trip an ``OVERSEER_ALERT`` (anomaly ``stuck-phase-transition``) on - the configured idle budget (env ``EGG_BRC_IDLE_BUDGET_MIN``, - default 30 min, architect od-4); priority climbs to ``high`` on the - 2× boundary; the loop keeps blocking rather than exiting 1 → - FAILED. -* escalate a consecutive agent-invocation failure streak (#3138): - linear backoff on the ``propose|ack|nack`` arm (streak × 2 s, - capped 30 s, parity with the ``confirm`` arm), a sticky log warning - at streak 5, and a sticky ``OVERSEER_ALERT`` (anomaly - ``agent-invocation-fail-streak``) at streak 10 whose detail - classifies sub-second failures as configuration-class (unknown - model alias, auth, prompt-rendering crash). The loop still never - self-FAILs. - -Slice-4 history -~~~~~~~~~~~~~~~ -* slice-4 task-4-1 flipped the unset-env defaults: ``EGG_BRC_EVENT_PUMP`` - from off→on, ``EGG_BRC_MEMORY`` from ``off``→``full``. -* slice-4 task-4-2 (this PR) deleted the legacy capped-restart - template, the recovery system / user prompts, the SSE consensus- - reached curl path, the legacy restart cap constant (issue #2806) - and its companion tunables (ready-poll cycles, transient-restart - backoff initial, startup-failure window seconds), the - ``EGG_BRC_EVENT_PUMP`` env flag itself, the legacy-template branch - in ``build_consensus_wrapped_command``, and the agent-side - heartbeat / gateway-session keep-alive in - ``sandbox/egg_agent_tools/handlers/message.py``. The event-pump - template is the only production path post-slice-4. Rollback under - a regression is a ``git revert`` of slices 1–3 per the PR body; - there is no env-flag rollback path. -* The buffer-overflow / transient-crash / startup-failure shell - classifiers survived the deletion and now live inside the - event-pump template (kept as named helpers for future use even - though the current ``propose|ack|nack`` arm relies on the - consecutive-failure counter + idle-budget escalation rather than - branching on them directly). +ONE-SHOT EVENT MODEL (#3164) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The orchestrator owns the BRC event loop (``orchestrator/event_loop.py``) +and spawns this wrapper ONCE per actionable BRC event, injecting the +event identity via env (``EGG_EVENT_ACTION`` ∈ ``propose|ack|nack``, +``EGG_EVENT_DEDUPE_KEY``, payload refs). The wrapper does no waiting of +its own — it re-derives ``egg-orch brc next-action`` once as a +stale-event backstop, invokes the agent exactly once via +``python3 -m egg_agent`` with the per-event prompt composed by +``orchestrator/routes/event_prompt.py:compose_event_prompt``, and exits +with the agent's rc. The orchestrator-side supervisor / convergence-stall +notifier (``event_loop.py``) owns respawn, backoff, idle-budget alerting, +and the failure-streak escalation that the retired in-pod loop carried. + +History +~~~~~~~ +* #2908 introduced the in-pod event-pump: a deterministic bash loop that + blocked on ``egg-orch message wait-loop`` between events while a 30 s + background heartbeat refreshed the slice-scoped gateway session. +* #3064/#3229 added the orchestrator-owned event loop behind an + ``EGG_EVENT_LOOP_OWNER`` flag (dormant one-shot arm spliced into the + wrapper) and proved it on a live BRC cycle. +* #3164 (this change) retired the in-pod wait arm, the 30 s background + heartbeat, the wrapper-side idle-budget alert, the failure-streak + escalation, and the ownership flag itself. The one-shot event handler + is now the only path — there is no in-pod loop and no rollback flag. """ import shlex -try: - from orchestrator import supervision_policy as _supervision_policy -except ImportError: - import supervision_policy as _supervision_policy # type: ignore[no-redef] - -# Export the #3138 constants from supervision_policy so the wrapper -# template can interpolate them via ``str.format`` — one source of truth -# for backoff / streak thresholds. -SUPERVISION_BACKOFF_FACTOR = _supervision_policy.SUPERVISION_BACKOFF_FACTOR -SUPERVISION_BACKOFF_CAP_SECONDS = _supervision_policy.SUPERVISION_BACKOFF_CAP_SECONDS -SUPERVISION_FAILURE_STREAK_WARN = _supervision_policy.SUPERVISION_FAILURE_STREAK_WARN -SUPERVISION_FAILURE_STREAK_ALERT = _supervision_policy.SUPERVISION_FAILURE_STREAK_ALERT - from egg_contracts.agent_roles import REVIEWER_CHECKOUT_ROLE_VALUES # Space-separated role values for which ``sync_to_proposals`` performs a @@ -81,167 +41,55 @@ EVENT_PUMP_CHECKOUT_ROLES = " ".join(sorted(str(r) for r in REVIEWER_CHECKOUT_ROLE_VALUES)) -def _event_loop_owner() -> str: - """Return the BRC event-loop ownership mode (``pod`` | ``orchestrator``). - - Thin lazy wrapper over ``env_config.get_event_loop_owner`` (#3064). - The dual-path import mirrors ``concurrent_executor`` / - ``global_slice_admit``: the orchestrator process may have the repo - root on ``sys.path`` (``orchestrator.env_config``) or only - ``orchestrator/`` itself (bare ``env_config``). Reading at - composition time keeps the one-shot-arm splice driven entirely by - the flag, so slice-1 needs no caller changes (dormant by design). - """ - try: - from orchestrator.env_config import get_event_loop_owner - except ImportError: - from env_config import get_event_loop_owner # type: ignore[no-redef,import-not-found] - return get_event_loop_owner() - - -# Default idle budget for the event-pump template (#2908 task-2-3). The -# overseer alert fires when ``LAST_PROGRESS`` ages past this many -# minutes without an actionable BRC event; priority climbs to ``high`` -# on the 2x boundary. The runtime override is the ``EGG_BRC_IDLE_BUDGET_MIN`` -# env var which the bash reads (composition-time formatting only sets the -# fallback when the env is unset/empty). 30 min is the architect od-4 -# default -- well above the WS7-observed 10-13 min legitimate-idle ceiling. +# Default idle budget for the BRC convergence-stall alert (#2908 od-4). +# The in-pod wrapper that originally owned this was retired in #3164; the +# orchestrator-side convergence-stall check (``event_loop.py``) now owns +# the alert, but the default lives here as the single source the health +# monitor's ``_IDLE_BUDGET_MIN_DEFAULT`` and the event loop both read. +# 30 min is well above the WS7-observed 10-13 min legitimate-idle ceiling. EVENT_PUMP_IDLE_BUDGET_MIN_DEFAULT = 30 -# OVERSEER_ALERT anomaly name raised when the idle budget is exceeded. The -# orchestrator-side re-homed convergence-stall check (``event_loop.py``) must -# raise the SAME name so the overseer classifies the in-pod and orchestrator -# variants identically. Single-sourced here and interpolated into the wrapper -# template below; imported by ``event_loop`` for its OVERSEER_ALERT emission. +# OVERSEER_ALERT anomaly name raised when the idle budget is exceeded. +# Single-sourced here and imported by ``event_loop`` for its +# orchestrator-side convergence-stall OVERSEER_ALERT emission. EVENT_PUMP_IDLE_BUDGET_ANOMALY = "stuck-phase-transition" -# Heartbeat cadence for the wrapper-owned background heartbeat emitter -# (#2908 task-2-2). Migrated from -# ``sandbox/egg_agent_tools/handlers/message.py:_WAIT_LOOP_HEARTBEAT_INTERVAL_SECS`` -# (60 s); 30 s keeps the wrapper well under the overseer's 120 s -# (default) / 600 s (implement-phase) ``heartbeat_threshold`` even on -# a one-missed-tick basis. Tests can override via -# ``EGG_BRC_HEARTBEAT_INTERVAL_SECS``. -EVENT_PUMP_HEARTBEAT_INTERVAL_SECS_DEFAULT = 30 - -# Inner wait-loop timeout for the event-pump's blocking call (#2908 -# task-2-1). Short relative to the idle budget so the bash loop returns -# regularly and can recompute next-action / age the idle counter. The -# orchestrator long-poll caps at 60 s anyway (see -# ``sandbox/egg_lib/orch_cli.py`` cmd_message_wait_loop), so matching -# that here avoids carrying a longer client timeout than the server -# honors. -EVENT_PUMP_WAIT_TIMEOUT_SECS_DEFAULT = 60 - - -# Event-pump bash template (#2908 task-2-1). Composed by -# ``build_consensus_wrapped_command`` — the only template path -# post-slice-4 task-4-2 (the legacy capped-restart template and the -# ``EGG_BRC_EVENT_PUMP`` env-flag read were deleted in that task). -# The pump is a deterministic loop that calls ``egg-orch brc get-state`` -# + ``egg-orch brc next-action`` to decide what to do next, blocks on -# ``egg-orch message wait-loop`` while emitting wrapper-owned heartbeats -# (#2036 + #2451 migrated out of the agent-side ``message_wait_loop`` -# handler), and invokes the agent one-shot via ``python3 -m egg_agent`` -# when an event needs handling. The wait-filter set is built -# conditionally with ``CONSENSUS_CONFIRMED`` omitted pre-confirm (the -# orchestrator rejects that combination with HTTP 400 per #2064 / #2482, -# risk_analyst R12). -# -# The idle budget (env ``EGG_BRC_IDLE_BUDGET_MIN``, default 30 min) -# replaces the legacy capped-restart cap: no actionable event for -# the budget duration raises an ``OVERSEER_ALERT``, but the loop -# keeps blocking instead of exiting 1 -> FAILED. +# One-shot event wrapper bash template (#3164). Composed by +# ``build_consensus_wrapped_command``. The orchestrator owns the BRC +# event loop and spawns this wrapper once per actionable event with +# ``EGG_EVENT_ACTION`` injected; the wrapper re-derives next-action once +# as a stale-event backstop, invokes the agent one-shot, and exits with +# the agent's rc. There is no blocking wait-loop, no background +# heartbeat, and no idle budget — the orchestrator owns all waiting and +# the convergence-stall / failure-streak escalation (see +# ``event_loop.py``). # # Placeholders interpolated by ``str.format``: # {agent_command_prefix} -- ``python3 -m egg_agent --model X --max-turns N`` -# {idle_budget_min_default}, {hb_interval_default}, {wait_timeout_default} -# {idle_budget_anomaly} -- OVERSEER_ALERT anomaly name on budget breach +# {checkout_roles} -- space-separated roles whose ack/nack arm +# performs a working-tree merge (#3216) _EVENT_PUMP_WRAPPER_TEMPLATE = r"""#!/bin/bash set -uo pipefail -# Event-pump wrapper (#2908 slice-2). Deterministic loop driven by -# ``egg-orch brc get-state`` + ``egg-orch brc next-action``; the agent -# is invoked one-shot per actionable event rather than holding a -# blocking wait. Migrated wrapper-owned heartbeats (#2036, #2451) -# replace the agent-side liveness path. - -IDLE_BUDGET_MIN="${{EGG_BRC_IDLE_BUDGET_MIN:-{idle_budget_min_default}}}" -IDLE_BUDGET_SECS=$(( IDLE_BUDGET_MIN * 60 )) -HB_INTERVAL_SECS="${{EGG_BRC_HEARTBEAT_INTERVAL_SECS:-{hb_interval_default}}}" -WAIT_TIMEOUT_SECS="${{EGG_BRC_WAIT_TIMEOUT_SECS:-{wait_timeout_default}}}" - -# Wrapper-owned background heartbeat PID. ``cleanup`` (installed below) -# kills it on EXIT so SIGTERM from the orchestrator does not leave a -# stray background process holding the gateway session open. -HB_BG_PID="" +# One-shot event wrapper (#3164). The orchestrator spawns this wrapper +# once per actionable BRC event (``EGG_EVENT_ACTION`` injected); the +# helper functions below are shared by the one-shot event handler +# appended at the end. No blocking wait-loop and no background heartbeat: +# the orchestrator owns all waiting. cw_log() {{ echo "[event-pump] $*" >&2 }} -# --- Agent-invocation exit-code classifiers (#2908 task-4-2) ----------- -# -# Migrated from the legacy capped-restart template. The event-pump -# invokes the agent one-shot per actionable event (``propose|ack|nack`` -# arm in the loop below); a non-zero exit there is still meaningful in -# the same three ways the legacy template distinguished: -# -# * ``is_buffer_overflow`` — the Claude Agent SDK 1 MiB JSON -# message-reader overflow signature (issue #2804). Deterministic -# under one-shot too: the next invocation hits the same oversized -# tool result. The event-pump's idle-budget safety net catches this -# eventually, but the classifier lets the operator alert sooner with -# a more specific anomaly tag. -# * ``is_transient_crash`` — signal-based exits (SIGABRT, SIGFPE, -# SIGKILL/OOM, SIGSEGV, Bun segfault) where a retry is appropriate. -# * ``is_startup_failure`` — exit 1 within the -# ``$STARTUP_FAILURE_WINDOW_SECONDS`` window (SDK API/network blip -# manifesting as exit 1 before the agent did meaningful work). -# -# The event-pump's ``propose|ack|nack`` arm does NOT yet branch on -# these signals (the consecutive-failure counter + idle-budget alert -# combo handles the operator-visible escalation today). Keeping them as -# named helpers preserves a clean hook for future revisions (per task-4-2 -# acceptance: classifiers "are still valid signals under the new idle/ -# no-progress safety budget"). -STARTUP_FAILURE_WINDOW_SECONDS=30 - -is_buffer_overflow() {{ - [ -f "${{AGENT_OUTPUT_LOG:-}}" ] || return 1 - grep -q "exceeded maximum buffer size" "$AGENT_OUTPUT_LOG" 2>/dev/null -}} - -is_transient_crash() {{ - local code="$1" - case "$code" in - 134|136|137|139|255) return 0 ;; # SIGABRT, SIGFPE, SIGKILL/OOM, SIGSEGV, Bun segfault - *) return 1 ;; - esac -}} - -is_startup_failure() {{ - local code="$1" - local duration="$2" - if [ "$code" -ne 1 ]; then - return 1 - fi - if [ "$duration" -lt "$STARTUP_FAILURE_WINDOW_SECONDS" ]; then - return 0 - fi - return 1 -}} - -# ----------------------------------------------------------------------- - # Emit one heartbeat. The CLI's ``message heartbeat`` handler auto- # attaches ``slice_id`` from ``$EGG_SLICE_ID`` via # ``_maybe_attach_slice_id`` in -# ``sandbox/egg_agent_tools/handlers/_gateway.py`` -- this is the -# #2451 migration: every wrapper heartbeat refreshes the slice-scoped -# gateway session as a side effect. We also echo the slice tag in the -# ``--body`` so a snapshot test (#2908 task-2-6 (ii)) can grep for the -# slice_id propagation without intercepting the HTTP POST. +# ``sandbox/egg_agent_tools/handlers/_gateway.py``. The one-shot event +# handler emits a single foreground ping (no background emitter) so the +# slice-5 silent-mid-event tripwire keys on heartbeats from an active +# one-shot pod. We also echo the slice tag in the ``--body`` so a +# snapshot test can grep for the slice_id propagation without +# intercepting the HTTP POST. emit_heartbeat() {{ local state="$1" local body_text="$2" @@ -252,100 +100,17 @@ def _event_loop_owner() -> str: >/dev/null 2>&1 || true }} -# Start a background heartbeat emitter for the duration of a blocking -# call. Replaces ``handlers/message.py:_start_wait_loop_heartbeat`` -- -# the wrapper now owns this responsibility (#2908 task-2-2). -start_background_heartbeat() {{ - local body_text="$1" - ( - # Install a TERM trap that exits the subshell cleanly so the - # outer ``stop_background_heartbeat``'s ``kill $HB_BG_PID`` - # (default signal SIGTERM) reaps the child rather than - # deadlocking on ``wait``. The earlier ``trap '' TERM`` form - # MASKED the signal and caused a wait deadlock that hung the - # whole event-pump after the first wait-loop return - # (reviewer_concurrency v1 finding 1 / #2908 slice-2 NACK). - # ``set -uo pipefail`` (without ``-e``) does not propagate - # subshell failures to the parent; ``emit_heartbeat``'s - # ``|| true`` already swallows any CLI error -- so there is no - # signal-defense to install here. - trap 'exit 0' TERM - while true; do - sleep "$HB_INTERVAL_SECS" - emit_heartbeat "WAITING_FOR_EVENT" "$body_text" - done - ) & - HB_BG_PID=$! -}} - -stop_background_heartbeat() {{ - if [ -n "$HB_BG_PID" ]; then - # Use SIGTERM (default ``kill`` signal) so the subshell's - # ``trap 'exit 0' TERM`` exits the heartbeat loop cleanly, - # then ``wait`` reaps it without blocking. SIGKILL would - # work too but loses the chance to log a final exit code. - kill "$HB_BG_PID" 2>/dev/null || true - wait "$HB_BG_PID" 2>/dev/null || true - HB_BG_PID="" - fi -}} - -cleanup() {{ - stop_background_heartbeat -}} -trap cleanup EXIT TERM INT - -# Fetch the BRC consensus state. Returns ``{{}}`` on any failure so -# downstream parsers can short-circuit without a Python crash. -fetch_state() {{ - egg-orch brc get-state --json 2>/dev/null || echo "{{}}" -}} - -# Has the role for this pod already reached CONFIRMED in the BRC -# matrix? Used to decide whether to include CONSENSUS_CONFIRMED in -# the wait-filter set (risk_analyst R12, orchestrator HTTP-400 rule -# from #2064 / #2482). -role_is_confirmed() {{ - local state_json="$1" - echo "$state_json" | python3 -c " -import sys, json, os -role = os.environ.get('EGG_AGENT_ROLE', '') -try: - d = json.load(sys.stdin) -except Exception: - print('False'); sys.exit(0) -agents = (d.get('consensus') or {{}}).get('agents') or {{}} -print('True' if agents.get(role, {{}}).get('confirmed') else 'False') -" 2>/dev/null || echo "False" -}} - -# Has global consensus already completed? Lets the loop short-circuit -# (e.g. when another role's confirmation flipped is_complete after we -# CONFIRMED but before SIGTERM landed). -consensus_is_complete() {{ - local state_json="$1" - echo "$state_json" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) -except Exception: - print('False'); sys.exit(0) -print('True' if (d.get('consensus') or {{}}).get('is_complete') else 'False') -" 2>/dev/null || echo "False" -}} - # Ask the orchestrator route what to do next. Returns a JSON document # with ``action`` ('wait'|'propose'|'ack'|'nack'|'confirm'|'complete') -# and an optional ``event_payload``. +# and an optional ``event_payload``. The one-shot handler uses this once +# as a stale-event freshness re-check before invoking the agent. # -# Per #2908 task-2-1 (acceptance: "Wrapper handles 409 stale_version -# and 409 aggregated-NACK from ``brc next-action`` as event-pump -# signals (re-fetch state, re-invoke), NOT as transient crashes to -# retry with backoff."), HTTP 409 from the route is a signal to -# re-fetch state, not a crash. The CLI surfaces 409 as a non-zero -# exit code with an empty/invalid JSON; falling back to ``{{"action":"wait"}}`` -# lets the next loop iteration call ``brc get-state`` again, which -# observes the new state and emits the correct next action. +# HTTP 409 from the route (409 stale_version, 409 aggregated-NACK +# barrier) is a signal, not a crash: the CLI surfaces it as a non-zero +# exit code with an empty/invalid JSON. We fall back to +# ``{{"action":"wait"}}`` and propagate the original ``rc`` so the +# one-shot handler can tell an inconclusive re-check (exit 75) from a +# positively-derived action. fetch_next_action() {{ local out rc out=$(egg-orch brc next-action --role "${{EGG_AGENT_ROLE:-unknown}}" --json 2>/dev/null) @@ -354,19 +119,14 @@ def _event_loop_owner() -> str: echo "$out" return 0 fi - # Fallback so the main loop keeps blocking on the bus and re-derives - # state on the next iteration. The CLI returns non-zero for any of: - # 409 stale_version, 409 aggregated-NACK barrier (both expected - # event-pump signals -- the next loop's ``brc get-state`` observes - # the changed state), 5xx, transport failure. Log it so operators - # reading wrapper logs can distinguish "orchestrator returned 409" - # (benign, expected) from "transport unreachable" (worth checking) - # without correlating against the orchestrator's audit log - # (tester v1 non-blocker #3). We always echo the fallback JSON so - # the next-action parser doesn't crash; we propagate the original - # ``rc`` as the function's exit code (caller reads ``$?`` after the - # ``$(...)`` substitution to count the streak per reviewer §3). - cw_log "brc next-action returned rc=$rc / empty body; falling back to {{\"action\":\"wait\"}} and re-deriving state next loop." + # The CLI returns non-zero for any of: 409 stale_version, 409 + # aggregated-NACK barrier, 5xx, transport failure. Log it so + # operators reading wrapper logs can distinguish "orchestrator + # returned 409" (benign, expected) from "transport unreachable" + # (worth checking). We always echo the fallback JSON so the parser + # doesn't crash, and propagate the original ``rc`` as the function's + # exit code (the caller reads ``$?`` after the ``$(...)`` capture). + cw_log "brc next-action returned rc=$rc / empty body; falling back to {{\"action\":\"wait\"}}." echo '{{"action":"wait"}}' return "$rc" }} @@ -394,44 +154,6 @@ def _event_loop_owner() -> str: " "$field" 2>/dev/null || echo "" }} -# Build the typed wait-filter set. Pre-confirm waits MUST omit -# CONSENSUS_CONFIRMED -- the orchestrator rejects that filter -# combination with HTTP 400 (#2064, #2482, risk_analyst R12). The -# six-event set is the union the architect's -# ``verification_strategy.slice_2.i`` snapshot test pins. -build_wait_args() {{ - local include_confirmed="$1" - local args=( - --for CONSENSUS_PROPOSE - --for CONSENSUS_ACK - --for CONSENSUS_NACK - --for STATUS - --for CONSENSUS_RE_REVIEW - --for OVERSEER_ALERT - ) - if [ "$include_confirmed" = "True" ]; then - args+=( --for CONSENSUS_CONFIRMED ) - fi - args+=( --timeout "$WAIT_TIMEOUT_SECS" ) - printf '%s\n' "${{args[@]}}" -}} - -# Block on the orchestrator message bus for the next BRC event while -# the wrapper-owned background heartbeat emitter keeps the overseer's -# liveness tracker and the gateway-session idle timer happy. -wait_for_event() {{ - local include_confirmed="$1" - local hb_body="event-pump wait role=${{EGG_AGENT_ROLE:-?}}" - mapfile -t WAIT_ARGS < <(build_wait_args "$include_confirmed") - start_background_heartbeat "$hb_body" - emit_heartbeat "WAITING_FOR_EVENT" "$hb_body" - egg-orch message wait-loop "${{WAIT_ARGS[@]}}" --max-iterations 1 >/dev/null 2>&1 - local rc=$? - stop_background_heartbeat - emit_heartbeat "WORKING" "event-pump woke (rc=$rc)" - return $rc -}} - # Invoke the agent one-shot with the per-event prompt. Slice-3 # (TASK-3-1 / TASK-3-2) replaces the slice-2 stub with the full # ``compose_event_prompt`` payload: memory excerpt (when @@ -654,515 +376,126 @@ def _event_loop_owner() -> str: return 0 }} -# Idle / no-progress safety budget (#2908 task-2-3). Replaces the -# legacy capped-restart cap (deleted by task-4-2): if no actionable -# event arrives for the configured idle budget we raise an -# OVERSEER_ALERT but the loop keeps blocking (the legacy template -# would exit 1 -> FAILED at this point). -LAST_PROGRESS=$SECONDS -ALERTED_AT_BUDGET=false -ALERTED_AT_DOUBLE=false - -# Consecutive-failure counters for the action arms (reviewer §1). -# Used to apply linear backoff on the ``confirm`` arm and to surface -# a distinguishable log when an action is persistently failing. -# The counters are arm-cluster scoped: they reset to 0 at the top of the -# loop when the next-action transitions away from the arm they apply to -# (reviewer §2 follow-up). That way a long-ago confirm-failure streak -# doesn't pre-load the backoff for a fresh confirm attempt hours later -# after the role briefly transitioned through ``wait`` / ``propose``. -CONFIRM_FAIL_STREAK=0 -AGENT_FAIL_STREAK=0 -# Sticky latches for the agent-invocation fail streak (#3138). Like the -# ``NEXT_ACTION_ALERTED_*`` latches below they are wrapper-lifetime -# sticky (the streak counter itself resets via the arm-cluster dispatch -# at the top of the loop, but a re-fired warning after a brief recovery -# is noise, not signal). The 10-streak latch escalates to a real -# overseer alert -- a streak of fast-failing invocations is a strong -# permanent/configuration-class signal (unknown model alias, auth -# misconfiguration, prompt-rendering crash) that the operator should -# hear about from the wrapper itself, not minutes later via the -# overseer's generic heartbeat-silence anomaly. -AGENT_FAIL_ALERTED_5=false -AGENT_FAIL_ALERTED_10=false -# Consecutive failures from ``fetch_next_action`` (reviewer §3): used -# to surface a distinguishable "many consecutive 5xx/transport failures" -# log line so an unhealthy orchestrator is differentiable from a benign -# 409 stale_version that re-derives on the next loop. Latches are -# sticky for the wrapper lifetime so the log doesn't re-fire if the -# counter happens to land back on the threshold after a brief recovery. -NEXT_ACTION_FAIL_STREAK=0 -NEXT_ACTION_ALERTED_5=false -NEXT_ACTION_ALERTED_20=false - -raise_idle_alert() {{ - local idle="$1" - local priority="$2" - local summary_extra="$3" - # Snapshot the current BRC state for the alert detail so operators - # see the consensus.agents. matrix without having to query - # pipeline status separately. Plan TASK-2-3 acceptance line: - # "alert payload includes anomaly type, priority, current BRC - # state" (tester v1 non-blocker #2). - local brc_snapshot snapshot_input - # The naive ``echo $VAR_OR_EMPTY_JSON | python3 ...`` form using a - # parameter-expansion default containing literal braces is unsafe: - # bash parses the brace inside the default greedily and leaves a - # trailing literal close-brace appended to the expanded value, - # corrupting JSON when STATE_JSON is set (tester v2 NACK finding). - # Use a separate variable and an explicit empty-string check so - # bash never sees unbalanced braces inside a parameter expansion. - snapshot_input="${{STATE_JSON-}}" - if [ -z "$snapshot_input" ]; then - snapshot_input='{{}}' - fi - brc_snapshot=$(printf '%s' "$snapshot_input" | python3 -c " -import sys, json, os -role = os.environ.get('EGG_AGENT_ROLE', '') -try: - d = json.load(sys.stdin) -except Exception: - print('(unavailable)'); sys.exit(0) -agents = (d.get('consensus') or {{}}).get('agents') or {{}} -my = agents.get(role) or {{}} -blocking = (d.get('consensus') or {{}}).get('blocking_agents') or [] -print(f\"role={{role}} producer_phase={{my.get('producer_phase','?')}} reviewer_phase={{my.get('reviewer_phase','?')}} confirmed={{my.get('confirmed','?')}} blocking_agents={{blocking}}\") -" 2>/dev/null || echo "(snapshot unavailable)") - timeout 5 egg-orch overseer alert "${{EGG_PIPELINE_ID:-unknown}}" \ - --role "${{EGG_AGENT_ROLE:-agent}}" \ - --anomaly {idle_budget_anomaly} \ - --priority "$priority" \ - --summary "BRC event-pump idle for ${{idle}}s$summary_extra" \ - --detail "Event-pump for role=${{EGG_AGENT_ROLE:-agent}} slice=${{EGG_SLICE_ID:-none}} has seen no actionable BRC event for ${{idle}}s (configured budget ${{IDLE_BUDGET_SECS}}s). The loop continues blocking; no FAILED transition is forced. BRC state: $brc_snapshot" \ - >/dev/null 2>&1 || true -}} - -raise_agent_fail_alert() {{ - local streak="$1" - local last_rc="$2" - local last_secs="$3" - local action="$4" - # Duration-aware classification (#3138): a fast failure means the - # invocation died before/at SDK init -- unknown model alias, auth - # misconfiguration, prompt-rendering crash -- i.e. a permanent - # configuration-class failure, not a transient. Name that in the - # alert so the operator doesn't have to infer it from cadence. The - # wrapper still never self-FAILs (post-#2908 design): the kill - # decision stays with the operator. - # - # The ``-le 2`` threshold (not ``< 1``) is deliberate: ``$SECONDS`` - # has whole-second granularity and ``invoke_agent_for_event`` includes - # prompt-composer overhead, so a genuine pre-SDK-init crash routinely - # measures 1-2s. Do not tighten this to ``< 1`` -- it would misclassify - # those crashes as transient. - local classification="repeated agent-invocation failure; attempts are taking ${{last_secs}}s so this may be transient (API/quota/transport)" - if [ "$last_secs" -le 2 ]; then - classification="attempts are fast-failing (${{last_secs}}s, before/at SDK init) -- likely a permanent configuration-class failure (unknown model alias, auth misconfiguration, prompt-rendering crash)" - fi - timeout 5 egg-orch overseer alert "${{EGG_PIPELINE_ID:-unknown}}" \ - --role "${{EGG_AGENT_ROLE:-agent}}" \ - --anomaly agent-invocation-fail-streak \ - --priority high \ - --summary "agent invocation failing repeatedly (action=${{action}}, streak=${{streak}})" \ - --detail "Event-pump for role=${{EGG_AGENT_ROLE:-agent}} slice=${{EGG_SLICE_ID:-none}} has had ${{streak}} consecutive agent-invocation failures on action=${{action}} (last rc=${{last_rc}}): ${{classification}}. The loop keeps retrying with linear backoff (capped 30s); no FAILED transition is forced. Idle budget continues to accrue." \ - >/dev/null 2>&1 || true -}} - -check_idle_budget() {{ - local idle=$(( SECONDS - LAST_PROGRESS )) - local double=$(( 2 * IDLE_BUDGET_SECS )) - if [ "$idle" -ge "$double" ] && [ "$ALERTED_AT_DOUBLE" != "true" ]; then - cw_log "Idle 2x budget exceeded (${{idle}}s >= ${{double}}s); raising HIGH overseer alert." - raise_idle_alert "$idle" "high" " (2x budget)" - ALERTED_AT_DOUBLE=true - # When the loop jumps straight from idle=0 to >=2x budget (e.g. - # the wrapper paused for 60+ min between checks), set the 1x - # latch too -- the 2x alert subsumes the 1x notification, so - # the next ``check_idle_budget`` should not re-fire the 1x - # branch (tester v1 non-blocker #1). - ALERTED_AT_BUDGET=true - elif [ "$idle" -ge "$IDLE_BUDGET_SECS" ] && [ "$ALERTED_AT_BUDGET" != "true" ]; then - cw_log "Idle budget exceeded (${{idle}}s >= ${{IDLE_BUDGET_SECS}}s); raising overseer alert." - raise_idle_alert "$idle" "high" "" - ALERTED_AT_BUDGET=true - fi -}} - -note_progress() {{ - LAST_PROGRESS=$SECONDS - ALERTED_AT_BUDGET=false - # Reviewer §6 nit: ``ALERTED_AT_DOUBLE`` is sticky for the lifetime - # of the loop. Once the operator has been paged at 2x budget, - # re-arming a fresh 1x alert later (after a single spurious - # ``note_progress``) is noise, not signal. -}} - -# --- main event-pump loop --- -cw_log "Event-pump starting (role=${{EGG_AGENT_ROLE:-?}}, slice=${{EGG_SLICE_ID:-none}}, idle-budget=${{IDLE_BUDGET_MIN}}m)" -emit_heartbeat "WORKING" "event-pump start" - -while true; do - STATE_JSON=$(fetch_state) - - if [ "$(consensus_is_complete "$STATE_JSON")" = "True" ]; then - cw_log "Global consensus complete; exiting cleanly." - exit 0 - fi - - ROLE_CONFIRMED=$(role_is_confirmed "$STATE_JSON") - - ACTION_JSON=$(fetch_next_action) - NEXT_ACTION_RC=$? - if [ "$NEXT_ACTION_RC" -eq 0 ]; then - NEXT_ACTION_FAIL_STREAK=0 - else - # Reviewer §3 (minor): the CLI surfaces 409 stale_version, 409 - # aggregated-NACK barrier, 5xx, and transport failure all as the - # same non-zero rc + empty body. A single non-zero rc is benign - # (expected event-pump signal). A *streak* is an orchestrator- - # health signal worth surfacing separately so operators reading - # wrapper logs can tell a 409-stuck role from an unhealthy - # orchestrator without correlating against the audit log. - # - # Use ``-ge`` with sticky latches (rather than ``-eq``) so the log - # is robust to any future change in how the counter advances and - # the warning fires the first time the threshold is crossed - # without re-firing on every iteration past it. - NEXT_ACTION_FAIL_STREAK=$(( NEXT_ACTION_FAIL_STREAK + 1 )) - if [ "$NEXT_ACTION_FAIL_STREAK" -ge {spvr_failure_streak_warn} ] && [ "$NEXT_ACTION_ALERTED_5" != "true" ]; then - cw_log "brc next-action has returned non-zero ${{NEXT_ACTION_FAIL_STREAK}} times in a row -- orchestrator may be unhealthy (5xx loop / transport down), not just a benign 409 stale_version. Idle budget continues to accrue." - NEXT_ACTION_ALERTED_5=true - fi - if [ "$NEXT_ACTION_FAIL_STREAK" -ge 20 ] && [ "$NEXT_ACTION_ALERTED_20" != "true" ]; then - cw_log "brc next-action has returned non-zero ${{NEXT_ACTION_FAIL_STREAK}} times in a row -- orchestrator may be unhealthy (5xx loop / transport down), not just a benign 409 stale_version. Idle budget continues to accrue." - NEXT_ACTION_ALERTED_20=true - fi - fi - ACTION=$(next_action_field "$ACTION_JSON" "action") - EVENT_PAYLOAD=$(next_action_field "$ACTION_JSON" "event_payload") - - # Reviewer §2 (non-blocking): the per-arm failure counters are - # arm-cluster scoped, not wrapper-lifetime. When the orchestrator - # transitions the role to a different action verb (e.g., a stuck - # ``confirm`` recovers via a fresh ``propose`` after a re-review - # event), reset the streak for the arm we just left so a brand-new - # attempt isn't pre-loaded with an old streak's backoff cap. - if [ "$ACTION" != "confirm" ]; then - CONFIRM_FAIL_STREAK=0 - fi - case "$ACTION" in - propose|ack|nack) ;; - *) AGENT_FAIL_STREAK=0 ;; - esac - - case "$ACTION" in - complete) - cw_log "Role complete; finalising via egg-orch consensus confirmed." - timeout 30 egg-orch consensus confirmed >/dev/null 2>&1 || true - cw_log "Exiting (role complete)." - exit 0 - ;; - confirm) - # Reviewer §1 (slice-4-blocking): ``note_progress`` must only - # fire when the CLI actually succeeded. Otherwise a persistent - # 5xx / transport / ``producer_not_fully_acked`` race against - # ``egg-orch consensus confirmed`` becomes a tight retry loop - # (~tens of ms per iteration, two short HTTP calls) that - # silently drains budget because the idle latch keeps resetting. - # The legacy template guarded this with a 3-restart cap; the - # event-pump path's equivalent is the idle-budget safety net - # gated on rc. - cw_log "Confirming via egg-orch consensus confirmed." - timeout 30 egg-orch consensus confirmed >/dev/null 2>&1 - confirm_rc=$? - if [ "$confirm_rc" -eq 0 ]; then - note_progress - CONFIRM_FAIL_STREAK=0 - else - # Floor the retry cadence so a persistent failure can't - # hot-loop the orchestrator faster than the idle counter - # can age. The sleep grows linearly with the streak length - # (capped at 30 s) so the operator sees the idle alert - # within ~30 min on the default budget while consecutive - # short failures don't escalate the load on the route. - CONFIRM_FAIL_STREAK=$(( CONFIRM_FAIL_STREAK + 1 )) - # NOTE: bash ``case``-branch scope is global, not function- - # local. Name reflects scope to avoid the false suggestion - # that this could be ``declare local`` (reviewer §4). - confirm_backoff_secs=$(( CONFIRM_FAIL_STREAK * {spvr_backoff_factor} )) - if [ "$confirm_backoff_secs" -gt {spvr_backoff_cap} ]; then - confirm_backoff_secs={spvr_backoff_cap} - fi - cw_log "consensus confirmed failed (rc=$confirm_rc, streak=$CONFIRM_FAIL_STREAK); backing off ${{confirm_backoff_secs}}s. Idle counter continues to accrue." - sleep "$confirm_backoff_secs" - fi - ;; - wait) - # Block on the bus. ``wait_for_event`` returns 0 only when - # a matching message was delivered (sandbox/egg_lib CLI - # contract: ``message wait-loop`` exits 0 on match, 1 on - # safety-cap / no-match). Reset the idle counter ONLY in - # the match case -- a timeout return is the DEFINITION of - # idle, not progress. (reviewer_concurrency v1 finding 2 / - # #2908 slice-2 NACK: unconditional ``note_progress`` here - # would defeat the entire idle-budget safety net because - # the inner wait-loop returns every ~60 s with no event.) - wait_for_event "$ROLE_CONFIRMED" - wait_rc=$? - if [ "$wait_rc" -eq 0 ]; then - note_progress - fi - ;; - propose|ack|nack) - # Reviewer §1 (slice-4-blocking): symmetric with the ``confirm`` - # arm above -- ``note_progress`` must only fire when the agent - # invocation actually succeeded. A persistent ``mcp__brc__propose`` - # / API-quota / prompt-rendering failure can fail in well under a - # second, and without rc-gating here the idle latch resets every - # iteration so the operator-visible idle alert never fires. The - # PR removed the legacy 3-restart cap; this rc gate is the - # equivalent ceiling on the action path. - cw_log "Invoking agent (action=$ACTION)." - # R11a (sync_to_proposals docstring anchor): the ``propose`` arm - # intentionally falls through this gate without syncing -- a - # producer's own commits are already on HEAD in its worktree, - # and merging peer proposal commits onto a producer turn is - # exactly the dual-role bleed-through the sync is designed to - # avoid. Only ``ack`` / ``nack`` (reviewer arms) sync. - if [ "$ACTION" = "ack" ] || [ "$ACTION" = "nack" ]; then - sync_to_proposals "$EVENT_PAYLOAD" - fi - agent_invoke_start=$SECONDS - invoke_agent_for_event "$ACTION" "$EVENT_PAYLOAD" - agent_rc=$? - agent_invoke_secs=$(( SECONDS - agent_invoke_start )) - if [ "$agent_rc" -eq 0 ]; then - note_progress - AGENT_FAIL_STREAK=0 - else - AGENT_FAIL_STREAK=$(( AGENT_FAIL_STREAK + 1 )) - cw_log "agent invocation failed (action=$ACTION, rc=$agent_rc, streak=$AGENT_FAIL_STREAK, duration=${{agent_invoke_secs}}s). Idle counter continues to accrue." - if [ "$AGENT_FAIL_STREAK" -ge {spvr_failure_streak_warn} ] && [ "$AGENT_FAIL_ALERTED_5" != "true" ]; then - # The per-failure line above already printed the streak - # count this iteration; this escalation line adds the - # diagnosis (likely permanent), not the count. - cw_log "agent invocation streak crossed {spvr_failure_streak_warn} (action=$ACTION) -- this is likely a permanent failure (unknown model alias, auth misconfiguration, prompt-rendering crash), not a transient. Idle budget continues to accrue." - AGENT_FAIL_ALERTED_5=true - fi - if [ "$AGENT_FAIL_STREAK" -ge {spvr_failure_streak_alert} ] && [ "$AGENT_FAIL_ALERTED_10" != "true" ]; then - raise_agent_fail_alert "$AGENT_FAIL_STREAK" "$agent_rc" "$agent_invoke_secs" "$ACTION" - AGENT_FAIL_ALERTED_10=true - fi - # #3138: the old flat ``sleep 1`` here assumed agent - # startup gives a natural seconds-to-tens-of-seconds - # floor, but a pre-SDK-init failure (unknown model - # alias, prompt-rendering crash) fails in <1 s and was - # retried at the floor indefinitely (160+ consecutive - # retries on the first issue-3077 run). Backoff parity - # with the ``confirm`` arm above: linear in the streak, - # capped at 30 s, so a deterministic fast-fail loop - # stops hammering the orchestrator while a one-off - # transient still retries promptly. - agent_backoff_secs=$(( AGENT_FAIL_STREAK * {spvr_backoff_factor} )) - if [ "$agent_backoff_secs" -gt {spvr_backoff_cap} ]; then - agent_backoff_secs={spvr_backoff_cap} - fi - sleep "$agent_backoff_secs" - fi - ;; - *) - # Defensive: unknown action surfaced from the orchestrator - # (older orchestrator, schema drift, transient error). Short - # sleep + recheck rather than a tight loop. The idle counter - # still ticks against the configured budget. - cw_log "Unknown next-action='$ACTION'; sleeping briefly and re-fetching." - sleep 5 - ;; - esac - - check_idle_budget -done """ -# Splice anchor: the one-shot arm (#3064 slice-1) is inserted immediately -# before this marker in the ORCHESTRATOR-ownership build of the wrapper. -# Splicing (rather than a ``str.format`` placeholder) is deliberate — a -# placeholder that expands to empty in pod mode would leave a residual -# newline and break the "pod build byte-identical to main" golden-file -# test. In pod mode the template is returned untouched. -_MAIN_LOOP_MARKER = "# --- main event-pump loop ---" - - -# One-shot event arm (#3064 slice-1). Spliced into the wrapper ONLY when -# the orchestrator owns the event loop (``EGG_EVENT_LOOP_OWNER= -# orchestrator``); the pod-ownership build never contains it and is -# byte-identical to the pre-#3064 template. Dormant by design: nothing -# sets ``EGG_EVENT_ACTION`` until the slice-2 spawner exists. -# -# This block is spliced AFTER ``_EVENT_PUMP_WRAPPER_TEMPLATE.format(...)`` -# runs, so it must use LITERAL braces (no ``{{``/``}}`` doubling) and must -# NOT be passed through ``str.format``. It relies only on helper functions -# defined above the marker (``fetch_next_action``, ``next_action_field``, -# ``sync_to_proposals``, ``invoke_agent_for_event``, ``emit_heartbeat``, -# ``cw_log``) and on the ``trap cleanup EXIT TERM INT`` already installed. +# One-shot event handler (#3164). Appended verbatim after +# ``_EVENT_PUMP_WRAPPER_TEMPLATE.format(...)`` runs, so it uses LITERAL +# braces (no ``{{``/``}}`` doubling) and is NOT passed through +# ``str.format``. It relies on the helper functions defined above +# (``fetch_next_action``, ``next_action_field``, ``sync_to_proposals``, +# ``invoke_agent_for_event``, ``emit_heartbeat``, ``cw_log``). # -# Behavior (task-1-1 acceptance): -# * Engages only when owner==orchestrator AND an event is injected -# (``EGG_EVENT_ACTION`` non-empty). With no injected event it falls -# through to the in-pod loop unchanged (belt-and-suspenders). -# * ``confirm``/``complete`` are executed orchestrator-side with no pod -# and must NEVER be injected here — loud non-zero rejection if they -# are. -# * Skips the blocking wait-loop and the background heartbeat entirely. -# * Re-checks next-action ONCE as a stale-event backstop. Three outcomes, -# three distinct exit codes so the slice-3 supervisor can tell them -# apart: -# - re-check inconclusive (``fetch_next_action`` rc != 0: 409 / -# 5xx / transport) -> exit 75 (EX_TEMPFAIL) WITHOUT invoking; the -# supervisor must re-derive rather than assume a clean handoff. -# - re-check succeeded but the derived action moved on -> exit 0 -# WITHOUT invoking (confirmed-stale dedupe backstop). -# - re-check confirms the event -> invoke ``invoke_agent_for_event`` -# exactly once and exit with the agent's (#2908-classified) rc. -_ONE_SHOT_ARM_TEMPLATE = r"""# --- one-shot event arm (#3064 slice-1, orchestrator ownership) --------- -# -# The orchestrator spawns this wrapper ONCE per actionable BRC event, -# injecting the event identity via env (EGG_EVENT_ACTION in -# propose|ack|nack, EGG_EVENT_DEDUPE_KEY, payload refs). In that mode we -# skip the blocking wait-loop and the background heartbeat: re-check -# next-action once as a stale-event backstop, invoke the agent exactly -# once, and exit with the agent's (#2908-classified) rc. The re-check has -# a distinct exit code (75/EX_TEMPFAIL) for an inconclusive re-check so a -# transient blip is never reported as a clean handoff. The -# orchestrator-side supervisor (slice-3) owns respawn/backoff/alerting. -ONE_SHOT_OWNER="${EGG_EVENT_LOOP_OWNER:-pod}" +# The orchestrator spawns this wrapper ONCE per actionable BRC event with +# ``EGG_EVENT_ACTION`` (in propose|ack|nack) injected. The handler: +# * Refuses loudly (exit 64) if no action was injected -- under the +# orchestrator-owned loop every pod is a one-shot event spawn, so a +# missing action is a caller bug, not a fall-through to an in-pod loop +# (which no longer exists). +# * Refuses ``confirm``/``complete`` (exit 64) -- those run +# orchestrator-side with no pod and must never reach a wrapper. +# * Re-checks next-action ONCE as a stale-event backstop, with three +# distinct exit codes the orchestrator supervisor reads: +# - inconclusive re-check (fetch rc != 0) -> exit 75 (EX_TEMPFAIL) +# WITHOUT invoking; the supervisor re-derives. +# - derived action moved on -> exit 0 WITHOUT invoking (stale dedupe). +# - re-check confirms the event -> invoke once, exit the agent rc. +_ONE_SHOT_EVENT_HANDLER = r"""# --- one-shot event handler (#3164) ------------------------------------ ONE_SHOT_ACTION="${EGG_EVENT_ACTION:-}" -if [ "$ONE_SHOT_OWNER" = "orchestrator" ] && [ -n "$ONE_SHOT_ACTION" ]; then - cw_log "One-shot arm engaged (action=$ONE_SHOT_ACTION, dedupe=${EGG_EVENT_DEDUPE_KEY:-none}, role=${EGG_AGENT_ROLE:-?}, slice=${EGG_SLICE_ID:-none})." - - case "$ONE_SHOT_ACTION" in - confirm|complete) - # confirm/complete are agent-free and run orchestrator-side - # (no pod is ever spawned for them). Reaching the one-shot arm - # with one injected is a caller bug -- reject loudly so the - # orchestrator surfaces it rather than silently invoking an - # agent for a verb that must never invoke one. - cw_log "FATAL: action=$ONE_SHOT_ACTION must run orchestrator-side with no pod and must never be injected into the one-shot arm. Refusing (exit 64)." - exit 64 - ;; - propose|ack|nack) - : ;; - *) - cw_log "FATAL: unknown injected EGG_EVENT_ACTION='$ONE_SHOT_ACTION' (expected propose|ack|nack). Refusing (exit 64)." - exit 64 - ;; - esac - - # Single foreground liveness ping (NOT the background emitter): the - # slice-5 silent-mid-event tripwire keys on heartbeats from an active - # one-shot pod. This spawns no background process. - emit_heartbeat "WORKING" "one-shot event arm action=$ONE_SHOT_ACTION" - - # Stale-event backstop: re-derive next-action ONCE. If consensus has - # moved on (derived action != injected event) this spawn is a - # duplicate/stale delivery -- exit 0 WITHOUT invoking the agent. The - # slice-2 spawner dedupes on the derived event, but a race can still - # deliver a stale one; this is the in-pod backstop for that race. - ONE_SHOT_NEXT_JSON=$(fetch_next_action) - ONE_SHOT_FETCH_RC=$? - ONE_SHOT_DERIVED=$(next_action_field "$ONE_SHOT_NEXT_JSON" "action") - if [ "$ONE_SHOT_FETCH_RC" -ne 0 ]; then - # The re-check itself did NOT cleanly succeed. ``fetch_next_action`` - # returns its ``{"action":"wait"}`` fallback (not a positively - # derived state) on ANY non-zero ``egg-orch brc next-action`` rc -- - # 409 stale_version / aggregated-NACK barrier, 5xx, and transport - # failure alike -- so we cannot distinguish a genuinely stale event - # from a transient orchestrator/transport blip. Exiting 0 here would - # report a "clean handoff" and let a transient blip silently DROP a - # live event (reviewer non-blocking note, PR #3167). Instead exit 75 - # (EX_TEMPFAIL): a distinct code that tells the slice-3 supervisor - # the freshness re-check was inconclusive, so it must re-derive - # next-action itself rather than treat the event as handled. The - # 409-vs-5xx refinement (benign-stale vs retryable) is deferred to - # the slice-3 supervisor contract, which owns respawn/backoff once - # the CLI exposes distinct codes; until then, fail safe. - cw_log "One-shot re-check did not cleanly succeed (fetch rc=$ONE_SHOT_FETCH_RC); cannot confirm event freshness. Exiting 75 (EX_TEMPFAIL) so the supervisor re-derives rather than assuming a clean handoff." - exit 75 - fi - if [ "$ONE_SHOT_DERIVED" != "$ONE_SHOT_ACTION" ]; then - # Re-check succeeded and the live action has positively moved on: - # this is a genuinely stale/duplicate delivery. exit 0 here means - # "confirmed stale, no agent needed" -- NOT "could not tell". - cw_log "Injected event is stale (injected=$ONE_SHOT_ACTION, derived=${ONE_SHOT_DERIVED:-}); exiting 0 without invoking the agent." - exit 0 - fi - - # Use the freshly-derived event_payload (current truth) for the - # invocation. Reviewer arms (ack/nack) sync the pending proposal - # commits into the worktree first, exactly as the in-pod loop does - # (#3076/#3077); the producer ``propose`` arm intentionally does not - # (R11a -- a producer's own commits are already on HEAD). - ONE_SHOT_PAYLOAD=$(next_action_field "$ONE_SHOT_NEXT_JSON" "event_payload") - if [ "$ONE_SHOT_ACTION" = "ack" ] || [ "$ONE_SHOT_ACTION" = "nack" ]; then - sync_to_proposals "$ONE_SHOT_PAYLOAD" - fi +if [ -z "$ONE_SHOT_ACTION" ]; then + cw_log "FATAL: no EGG_EVENT_ACTION injected. Under the orchestrator-owned event loop (#3164) every agent pod is spawned one-shot per BRC event; a wrapper with no injected action is a caller bug. Refusing (exit 64)." + exit 64 +fi +cw_log "One-shot event handler engaged (action=$ONE_SHOT_ACTION, dedupe=${EGG_EVENT_DEDUPE_KEY:-none}, role=${EGG_AGENT_ROLE:-?}, slice=${EGG_SLICE_ID:-none})." + +case "$ONE_SHOT_ACTION" in + confirm|complete) + # confirm/complete are agent-free and run orchestrator-side (no + # pod is ever spawned for them). Reaching a wrapper with one + # injected is a caller bug -- reject loudly. + cw_log "FATAL: action=$ONE_SHOT_ACTION must run orchestrator-side with no pod and must never be injected into a wrapper. Refusing (exit 64)." + exit 64 + ;; + propose|ack|nack) + : ;; + *) + cw_log "FATAL: unknown injected EGG_EVENT_ACTION='$ONE_SHOT_ACTION' (expected propose|ack|nack). Refusing (exit 64)." + exit 64 + ;; +esac + +# Single foreground liveness ping (no background emitter): the +# silent-mid-event tripwire keys on heartbeats from an active one-shot pod. +emit_heartbeat "WORKING" "one-shot event handler action=$ONE_SHOT_ACTION" + +# Stale-event backstop: re-derive next-action ONCE. If consensus has moved +# on (derived action != injected event) this spawn is a duplicate/stale +# delivery -- exit 0 WITHOUT invoking the agent. The spawner dedupes on the +# derived event, but a race can still deliver a stale one; this is the +# in-pod backstop for that race. +ONE_SHOT_NEXT_JSON=$(fetch_next_action) +ONE_SHOT_FETCH_RC=$? +ONE_SHOT_DERIVED=$(next_action_field "$ONE_SHOT_NEXT_JSON" "action") +if [ "$ONE_SHOT_FETCH_RC" -ne 0 ]; then + # The re-check itself did NOT cleanly succeed. ``fetch_next_action`` + # returns its ``{"action":"wait"}`` fallback on ANY non-zero rc (409 + # stale_version / aggregated-NACK barrier, 5xx, transport failure + # alike), so we cannot distinguish a genuinely stale event from a + # transient blip. Exiting 0 here would report a "clean handoff" and let + # a transient blip silently DROP a live event. Instead exit 75 + # (EX_TEMPFAIL): a distinct code telling the orchestrator supervisor the + # freshness re-check was inconclusive, so it must re-derive next-action + # rather than treat the event as handled. + cw_log "One-shot re-check did not cleanly succeed (fetch rc=$ONE_SHOT_FETCH_RC); cannot confirm event freshness. Exiting 75 (EX_TEMPFAIL) so the supervisor re-derives rather than assuming a clean handoff." + exit 75 +fi +if [ "$ONE_SHOT_DERIVED" != "$ONE_SHOT_ACTION" ]; then + # Re-check succeeded and the live action has positively moved on: a + # genuinely stale/duplicate delivery. exit 0 means "confirmed stale, no + # agent needed" -- NOT "could not tell". + cw_log "Injected event is stale (injected=$ONE_SHOT_ACTION, derived=${ONE_SHOT_DERIVED:-}); exiting 0 without invoking the agent." + exit 0 +fi - one_shot_start=$SECONDS - invoke_agent_for_event "$ONE_SHOT_ACTION" "$ONE_SHOT_PAYLOAD" - one_shot_rc=$? - one_shot_secs=$(( SECONDS - one_shot_start )) - cw_log "One-shot invocation done (action=$ONE_SHOT_ACTION, rc=$one_shot_rc, duration=${one_shot_secs}s); exiting with the agent rc." - # Exit with the agent's (#2908-classified) rc. The classifiers - # (is_buffer_overflow / is_transient_crash / is_startup_failure) - # remain defined above for a future revision that wants to remap; the - # slice-3 orchestrator-side supervisor reads the Job exit code here. - # - # Exit-code contract for the slice-3 supervisor (reviewer note, PR - # #3167): 75/EX_TEMPFAIL is RESERVED for the inconclusive-re-check - # outcome above and means "freshness could not be confirmed, re-derive - # next-action". The #2908 classification codes the agent passthrough can - # emit here do NOT include 75, so the two meanings do not collide today; - # the slice-3 supervisor MUST keep 75 reserved for the re-check meaning - # and must not map any agent outcome onto it. - exit "$one_shot_rc" +# Use the freshly-derived event_payload (current truth) for the invocation. +# Reviewer arms (ack/nack) sync the pending proposal commits into the +# worktree first (#3076/#3077); the producer ``propose`` arm intentionally +# does not (R11a -- a producer's own commits are already on HEAD). +ONE_SHOT_PAYLOAD=$(next_action_field "$ONE_SHOT_NEXT_JSON" "event_payload") +if [ "$ONE_SHOT_ACTION" = "ack" ] || [ "$ONE_SHOT_ACTION" = "nack" ]; then + sync_to_proposals "$ONE_SHOT_PAYLOAD" fi +one_shot_start=$SECONDS +invoke_agent_for_event "$ONE_SHOT_ACTION" "$ONE_SHOT_PAYLOAD" +one_shot_rc=$? +one_shot_secs=$(( SECONDS - one_shot_start )) +cw_log "One-shot invocation done (action=$ONE_SHOT_ACTION, rc=$one_shot_rc, duration=${one_shot_secs}s); exiting with the agent rc." +# Exit-code contract for the orchestrator supervisor: 75/EX_TEMPFAIL is +# RESERVED for the inconclusive-re-check outcome above ("freshness could +# not be confirmed, re-derive next-action"). Agent passthrough rcs do not +# include 75, so the two meanings never collide. +exit "$one_shot_rc" """ -# ``_event_pump_enabled`` (the read of ``EGG_BRC_EVENT_PUMP``) was -# deleted in slice-4 task-4-2 along with the legacy template branch in -# ``build_consensus_wrapped_command``. The env flag is now silently -# inert; operators with it lingering in k8s manifests can remove it -# without any production impact. - - def build_event_pump_wrapped_command( prompt_text: str, model: str = "opus", max_turns: int = 1000, - idle_budget_min: int = EVENT_PUMP_IDLE_BUDGET_MIN_DEFAULT, - heartbeat_interval_secs: int = EVENT_PUMP_HEARTBEAT_INTERVAL_SECS_DEFAULT, - wait_timeout_secs: int = EVENT_PUMP_WAIT_TIMEOUT_SECS_DEFAULT, effort: str | None = None, ) -> list[str]: - """Compose the event-pump wrapper bash command (#2908 task-2-1). + """Compose the one-shot event wrapper bash command (#3164). - Public entry-point retained so tests and - ``build_consensus_wrapped_command`` (which now unconditionally - delegates here post slice-4 task-4-2) share one composer. + Public entry-point shared by ``build_consensus_wrapped_command``. - The ``prompt_text`` argument is kept for signature parity with - the legacy capped-restart entry-point that task-4-2 deleted; the - event-pump emits its own per-event prompts inside - ``invoke_agent_for_event`` from the rendered ``compose_event_prompt`` - output (slice-3 task-3-1), so the initial prompt is not - interpolated into the bash directly. A future revision could - choose to pass it through as a bootstrap prompt for the first - ``propose`` event without breaking the public signature. + The ``prompt_text`` argument is accepted for signature parity with the + historical entry-point; the wrapper emits its own per-event prompt + inside ``invoke_agent_for_event`` from the rendered + ``compose_event_prompt`` output, so the initial prompt is not + interpolated into the bash directly. """ - del prompt_text # interface parity with the deleted legacy entry-point + del prompt_text # interface parity; per-event prompts are composed in-wrapper agent_prefix_parts = [ "python3", @@ -1181,35 +514,15 @@ def build_event_pump_wrapped_command( script = _EVENT_PUMP_WRAPPER_TEMPLATE.format( agent_command_prefix=agent_command_prefix, - idle_budget_min_default=idle_budget_min, - idle_budget_anomaly=EVENT_PUMP_IDLE_BUDGET_ANOMALY, - hb_interval_default=heartbeat_interval_secs, - wait_timeout_default=wait_timeout_secs, - spvr_backoff_factor=SUPERVISION_BACKOFF_FACTOR, - spvr_backoff_cap=SUPERVISION_BACKOFF_CAP_SECONDS, - spvr_failure_streak_warn=SUPERVISION_FAILURE_STREAK_WARN, - spvr_failure_streak_alert=SUPERVISION_FAILURE_STREAK_ALERT, checkout_roles=EVENT_PUMP_CHECKOUT_ROLES, ) # The triple-quoted template opens with a newline (so the source reads - # cleanly); strip it so ``#!/bin/bash`` lands on line 1. The script is run - # via ``bash -c`` where the shebang is cosmetic, so pod runtime behaviour - # is unchanged — but a leading blank line trips shellcheck SC1128 on the - # committed golden snapshot, so we drop it at the single rendering source. + # cleanly); strip it so ``#!/bin/bash`` lands on line 1. script = script.lstrip("\n") - # #3064 slice-1: in orchestrator-ownership mode, splice the dormant - # one-shot arm in ahead of the main loop. In pod mode (default) the - # template is returned untouched so the generated wrapper is - # byte-identical to pre-#3064 (golden-file test). The arm itself only - # engages when an event is injected (EGG_EVENT_ACTION) -- nothing sets - # that until slice-2 -- so this is dormant even when orchestrator mode - # is selected without the spawner. - if _event_loop_owner() == "orchestrator": - script = script.replace( - _MAIN_LOOP_MARKER, - _ONE_SHOT_ARM_TEMPLATE + _MAIN_LOOP_MARKER, - 1, - ) + # The orchestrator owns the event loop (#3164): every spawn is one-shot, + # so the one-shot event handler is always appended. There is no in-pod + # wait loop and no ownership flag to branch on. + script = script.rstrip("\n") + "\n\n" + _ONE_SHOT_EVENT_HANDLER return ["bash", "-c", script] @@ -1219,22 +532,20 @@ def build_consensus_wrapped_command( max_turns: int = 1000, effort: str | None = None, ) -> list[str]: - """Build a shell command that runs the agent under the BRC event-pump wrapper. + """Build a shell command that runs the agent under the one-shot BRC wrapper. - Slice-4 task-4-2 collapsed this function to a thin alias for - :func:`build_event_pump_wrapped_command`. The legacy capped-restart - template, recovery system prompt, and ``EGG_BRC_EVENT_PUMP`` env - flag were all deleted; the event-pump template is the only - production path post-slice-4. The function signature is retained - so call sites in ``concurrent_executor.py`` and ``kubernetes_spawner.py`` - do not have to be renamed in lock-step with this slice. + A thin alias for :func:`build_event_pump_wrapped_command`, retained so + call sites in ``concurrent_executor.py`` and ``kubernetes_spawner.py`` + do not have to be renamed. The orchestrator owns the event loop + (#3164): the wrapper is spawned one-shot per BRC event and there is no + in-pod wait loop. Args: - prompt_text: Initial prompt (reserved — the event-pump emits - its own per-event prompts inside ``invoke_agent_for_event`` - from the rendered ``compose_event_prompt`` output, so the - initial prompt is not interpolated into the bash directly. - Accepted for interface parity with the legacy signature). + prompt_text: Initial prompt (reserved — the wrapper emits its own + per-event prompts inside ``invoke_agent_for_event`` from the + rendered ``compose_event_prompt`` output, so the initial prompt + is not interpolated into the bash directly. Accepted for + interface parity with the historical signature). model: Agent model to use. max_turns: Maximum number of tool-call turns per agent invocation. diff --git a/orchestrator/env_config.py b/orchestrator/env_config.py index a9e0b9adf6..36b35ce08c 100644 --- a/orchestrator/env_config.py +++ b/orchestrator/env_config.py @@ -231,23 +231,29 @@ def get_heartbeat_rate_limit() -> int: # #2137 — slice-scheduler configuration knobs. # # EGG_ORCH_MAX_PARALLEL_SLICES — soft concurrency cap on slice spawns -# per wave. Default 1 — a single slice runs at once unless explicitly -# raised. Each slice spawns ~8 agent containers, so a higher cap can -# saturate a small/single-node host (lowered to 1 after a 2-slice wave -# overwhelmed a single-node cluster). This env var is the fallback -# default; the authoritative per-pipeline knob is the +# per wave. Default 4. Under the orchestrator-owned event loop (#3164) +# a slice no longer pins ~8 resident agent containers for its whole +# lifetime — the orchestrator spawns at most one short-lived one-shot +# pod per actionable BRC event, so the host load of an in-flight slice +# is a fraction of the old resident-cohort cost. That is what makes a +# 4-wide default safe on a single-node host where the resident model +# forced the cap down to 1. This env var is the fallback default; the +# authoritative per-pipeline knob is the # ``PipelineConfig.max_parallel_slices`` field, set at pipeline # creation, which takes precedence when set. (History: original # decision-5 cap of 5; lowered to 2 in #2466 for container/gateway -# pressure; lowered to 1 for single-node host safety.) +# pressure; lowered to 1 for single-node host safety under the +# resident-pod model; raised to 4 in #3164 once on-demand spawning +# removed the resident-cohort cost.) # # EGG_ORCH_GLOBAL_MAX_PARALLEL_SLICES — orchestrator-process-wide # cap on slices in flight across ALL running pipelines (#2241 -# gap 1). Default 4 (operationally observed safe ceiling — each -# slice spawns ~8 containers and the host saturates beyond that). -# Slice spawn defers when saturated; admission happens before -# ``mark_spawned`` so the per-pipeline cap accounting stays -# honest. +# gap 1). Default 4. Under on-demand spawning (#3164) an in-flight +# slice costs at most one short-lived one-shot pod per BRC event +# rather than a resident ~8-container cohort, so a single 4-wide +# pipeline no longer saturates the host at this ceiling. Slice spawn +# defers when saturated; admission happens before ``mark_spawned`` so +# the per-pipeline cap accounting stays honest. # # EGG_ORCH_SLICE_LOCAL_MAX_CYCLES — per-slice BRC re-proposal ceiling # before HITL escalation (refine-phase decision-9 opt-3 two-tier @@ -268,7 +274,7 @@ def get_heartbeat_rate_limit() -> int: # decision-16 opt-3 hybrid). # ----------------------------------------------------------------- -DEFAULT_MAX_PARALLEL_SLICES = 1 +DEFAULT_MAX_PARALLEL_SLICES = 4 DEFAULT_GLOBAL_MAX_PARALLEL_SLICES = 4 DEFAULT_SLICE_LOCAL_MAX_CYCLES = 3 DEFAULT_SLICE_GLOBAL_MAX_CYCLES = 10 @@ -329,7 +335,7 @@ def _coerce_positive_float(env_name: str, default: float) -> float: def get_max_parallel_slices() -> int: - """Return the per-pipeline parallel-slice spawn cap (default 1).""" + """Return the per-pipeline parallel-slice spawn cap (default 4).""" return _coerce_positive_int("EGG_ORCH_MAX_PARALLEL_SLICES", DEFAULT_MAX_PARALLEL_SLICES) @@ -338,9 +344,10 @@ def get_global_max_parallel_slices() -> int: Distinct from ``EGG_ORCH_MAX_PARALLEL_SLICES`` which caps slices per pipeline. This cap bounds slices across **all** running - pipelines in the orchestrator process — the operationally - observed safe ceiling is ~4 because each slice spawns ~8 - containers and the host saturates beyond that. + pipelines in the orchestrator process. Under on-demand spawning + (#3164) an in-flight slice costs at most one short-lived one-shot + pod per BRC event rather than a resident ~8-container cohort, so 4 + remains a comfortable ceiling rather than a saturation point. Slice spawn defers (the slice stays READY) when the cap is saturated; the per-pipeline ``iter_ready`` accounting is not @@ -470,51 +477,7 @@ def get_recovery_ref_cleanup_interval_seconds() -> float: ) -# ----------------------------------------------------------------- -# EGG_EVENT_LOOP_OWNER — who drives the BRC event loop (#3064). -# -# ``pod`` (default) — today's behavior: each agent container runs -# the in-pod event-pump wait-loop in ``consensus_wrapper.py``. -# ``orchestrator`` — the orchestrator owns the loop and spawns a -# one-shot pod per actionable BRC event (slice-2+). The wrapper's -# dormant one-shot arm (slice-1) only engages in this mode. -# -# Unlike the other knobs in this module, an unrecognised value is -# rejected LOUDLY (``ValueError``) rather than silently falling back to -# the default. There is no safe silent fallback for an ownership mode: -# the #3023 post-mortem showed that getting this wrong either deadlocks -# BRC (loop silenced with nothing replacing it) or spawns duplicate -# pods, so a typo must surface immediately instead of masquerading as -# the default. The default-when-unset path stays ``pod`` so the flag is -# dormant until an operator opts in. -# ----------------------------------------------------------------- - -EVENT_LOOP_OWNER_POD = "pod" -EVENT_LOOP_OWNER_ORCHESTRATOR = "orchestrator" -DEFAULT_EVENT_LOOP_OWNER = EVENT_LOOP_OWNER_POD -VALID_EVENT_LOOP_OWNERS = (EVENT_LOOP_OWNER_POD, EVENT_LOOP_OWNER_ORCHESTRATOR) - - -def get_event_loop_owner() -> str: - """Return the BRC event-loop ownership mode (default ``pod``). - - Reads ``EGG_EVENT_LOOP_OWNER`` (case-insensitive) ∈ - {``pod``, ``orchestrator``}. Unset/empty ⇒ ``pod`` (the in-pod - wait-loop, today's behavior). Any other value raises ``ValueError`` - — there is no safe silent default for an ownership mode, so a - misconfiguration fails loudly at read time (see the module comment - and the #3023 post-mortem). - """ - raw = os.environ.get("EGG_EVENT_LOOP_OWNER", "").strip() - if not raw: - return DEFAULT_EVENT_LOOP_OWNER - normalized = raw.lower() - if normalized not in VALID_EVENT_LOOP_OWNERS: - msg = ( - f"EGG_EVENT_LOOP_OWNER={raw!r} is not a recognised ownership " - f"mode; valid values are {VALID_EVENT_LOOP_OWNERS} " - f"(default {DEFAULT_EVENT_LOOP_OWNER!r})." - ) - logger.error(msg) - raise ValueError(msg) - return normalized +# EGG_EVENT_LOOP_OWNER and ``get_event_loop_owner`` were removed in #3164. +# The orchestrator now unconditionally owns the BRC event loop (one-shot +# pod per actionable event); the in-pod wait-loop and the ownership flag +# it gated were retired after the live proving run on #3200. diff --git a/orchestrator/event_loop.py b/orchestrator/event_loop.py index 5120e128aa..a9064c9e37 100644 --- a/orchestrator/event_loop.py +++ b/orchestrator/event_loop.py @@ -1,14 +1,14 @@ """Orchestrator-owned BRC event loop (#3064 slice-2, TASK-2-1; slice-3, TASK-3-1; slice-5, TASK-5-1). -Under ``EGG_EVENT_LOOP_OWNER=orchestrator`` the orchestrator — not a -long-lived in-pod wait-loop — owns the BRC event loop. For every role it -consumes the logic backing ``routes.consensus._derive_next_action`` -IN-PROCESS and maps the derived verb onto a lifecycle action: +The orchestrator — not a long-lived in-pod wait-loop — owns the BRC +event loop (unconditionally, post-#3164). For every role it consumes the +logic backing ``routes.consensus._derive_next_action`` IN-PROCESS and +maps the derived verb onto a lifecycle action: * ``propose | ack | nack`` → request a one-shot Job via an injectable - spawner (the pod handles exactly one event and exits — the slice-1 - wrapper arm). + spawner (the pod handles exactly one event and exits — the wrapper's + one-shot event handler). * ``confirm | complete`` → executed orchestrator-side, agent-free, with **no pod** (mirrors the wrapper's ``egg-orch consensus confirmed`` arm). * ``wait`` → nothing. diff --git a/orchestrator/health_monitor.py b/orchestrator/health_monitor.py index bc31982494..f6ecdc538d 100644 --- a/orchestrator/health_monitor.py +++ b/orchestrator/health_monitor.py @@ -145,28 +145,16 @@ def __init__( # BRC progress tracking: first time each fully-ACKed producer was seen self._fully_acked_first_seen: dict[str, float] = {} - # #3064 slice-5: orchestrator-mode awareness. In orchestrator mode: - # * roles with no active Job are normal (never alert on heartbeat - # timeout or container-exit). - # * heartbeat / progress / container-exit tripwires apply only - # when a Job is active for that role. - # * a silent one-shot pod mid-event still trips — the role IS - # active so tripwires fire normally. - # Auto-detect orchestrator mode from the env config. This lets the - # monitor configure itself at construction time without requiring - # an explicit set_orchestrator_mode() call from every caller — the - # concurrent executor also calls set_orchestrator_mode(True) during - # event-loop setup as a belt-and-suspenders production path. - try: - from env_config import get_event_loop_owner - - self._orchestrator_mode: bool = get_event_loop_owner() == "orchestrator" - except Exception: # noqa: BLE001 — best-effort auto-detect - self._orchestrator_mode: bool = False - # Set of roles with an active one-shot Job. Populated externally - # (the concurrent executor or event loop sets this from live Job - # labels on each poll). Empty/unset means "unknown — fall through - # to pod-mode alerting behavior". + # #3164: the orchestrator unconditionally owns the BRC event loop. + # * roles with no active one-shot Job are legitimately idle (the + # loop has not derived an actionable event for them yet) and are + # skipped from heartbeat/progress/container-exit tripwires. + # * a role WITH an active Job is checked normally — a silent + # one-shot pod mid-event still trips. + # ``_active_jobs`` is the set of roles with an active one-shot Job, + # populated by the event loop on each poll tick via + # ``set_active_roles``. Empty means "no active Jobs — every role is + # idle", so all tripwires are skipped. self._active_jobs: set[str] = set() # Subscribe to events @@ -209,21 +197,6 @@ def get_current_phase(self) -> str | None: with self._lock: return self._current_phase - def set_orchestrator_mode(self, enabled: bool = True) -> None: - """Enable or disable orchestrator-mode alert suppression. - - In orchestrator mode (``enabled=True``), roles with no active Job - are normal — heartbeat/progress/container-exit tripwires apply only - while that role's Job is active. A silent one-shot pod mid-event - still alerts. Pod mode (``enabled=False``, the default) restores - today's behavior where every registered agent's silence can trip - alerts. - - Thread-safe. - """ - with self._lock: - self._orchestrator_mode = enabled - def set_active_roles(self, roles: set[str]) -> None: """Set the set of roles with currently active one-shot Jobs. @@ -462,22 +435,17 @@ def _has_recent_peer_progress( return False, None def _orchestrator_skip_tripwire(self, agent_id: str) -> bool: - """Return True if this agent should be skipped by tripwire checks in orchestrator mode. + """Return True if this agent should be skipped by tripwire checks. - In orchestrator mode, a role with no active one-shot Job is - legitimately idle — the orchestrator event loop simply has not - derived an actionable event for it yet. Alerting on such a role - would be a false positive. + The orchestrator owns the BRC event loop (#3164): a role with no + active one-shot Job is legitimately idle — the loop simply has not + derived an actionable event for it yet — so alerting on it would be + a false positive. An empty ``_active_jobs`` (default from init or explicitly set via ``set_active_roles(set())``) means there are zero active one-shot Jobs — every role is legitimately idle; all tripwires are skipped. - This is safe because ``pod`` mode returns early before this check, - so empty ``_active_jobs`` only reaches this logic when we are - demonstrably in orchestrator mode. """ - if not self._orchestrator_mode: - return False # Empty active_jobs (no active one-shot Jobs) → every role is idle if not self._active_jobs: return True diff --git a/orchestrator/heartbeat.py b/orchestrator/heartbeat.py index 4c4de465cb..d3ebe206ee 100644 --- a/orchestrator/heartbeat.py +++ b/orchestrator/heartbeat.py @@ -48,11 +48,12 @@ class HeartbeatCoordinator: Thread-safe. One instance lives as an orchestrator-process-global singleton (``get_heartbeat_coordinator``). - Mode guard (#3064 slice-5): in orchestrator mode the gateway-session - refresh side-effect (``should_fan_out_gateway_session``) is suppressed - — session refresh happens at spawn time (slice-4 worktree re-attach), - not via heartbeat fan-out. Absent-sender alerts are handled by the - health monitor's mode-aware tripwires, not the coordinator. + The coordinator validates, dedups, and rate-limits HEARTBEATs for + liveness tracking. It has no gateway-session side effect: under the + orchestrator-owned event loop (#3164) a pod is one-shot per BRC event, + so its gateway session is refreshed at spawn time (worktree re-attach), + not via a heartbeat fan-out (the resident-pod keep-alive that lived + here was removed in #3164). """ def __init__(self, window_seconds: int = 60) -> None: @@ -62,11 +63,6 @@ def __init__(self, window_seconds: int = 60) -> None: self._windows: dict[_Key, deque[float]] = {} # (pipeline_id, slice_id, role) -> last-seen (state, waiting_on) self._last_state: dict[_Key, tuple[str, str]] = {} - # (pipeline_id, slice_id, role) -> last gateway-session fan-out epoch-seconds - self._last_fan_out: dict[_Key, float] = {} - # #3064 slice-5: in orchestrator mode the heartbeat path does NOT - # refresh the gateway session — that happens at spawn time. - self._orchestrator_mode: bool = False def check_rate_limit( self, @@ -134,79 +130,13 @@ def record_state( with self._lock: self._last_state[key] = (state, waiting_on or "") - def should_fan_out_gateway_session( - self, - pipeline_id: str, - slice_id: str | None, - role: str, - min_interval_seconds: float, - ) -> bool: - """Throttle gateway-session fan-outs (issue #2076 NB2). - - The HEARTBEAT route fans out a gateway-session refresh both on - the dedup early-return path and after the rate-limit gate. The - dedup path bypasses the per-(slice, role) rate limiter (by - design — see ``check_rate_limit``'s NB1 from #1897), so a - misbehaving agent hot-looping with identical state can amplify - into the gateway without burning rate budget. This per-(slice, - role) cooldown caps that amplification independently of the - heartbeat-acceptance rate. - - Returns ``True`` and records the new timestamp if at least - ``min_interval_seconds`` have passed since the previous fan-out - for this ``(pipeline_id, slice_id, role)``; returns ``False`` - (without recording) if the caller should skip the fan-out this - round. Any non-positive ``min_interval_seconds`` (``<= 0``) - disables throttling — every call returns ``True`` without - recording. - - Callers must pass a finite real number. ``float('nan')`` falls - through both branches (``nan <= 0`` and ``now - last < nan`` are - both False), which would silently behave as "always record, - never suppress" — not a meaningful state. The realistic call - site (``_GATEWAY_FANOUT_MIN_INTERVAL_SECONDS = 30.0``) is a - module constant, but if a future env-var-driven knob lands - (#2076 NB5) it should sanitize NaN/inf at parse time. - """ - if min_interval_seconds <= 0: - return True - key: _Key = (pipeline_id, slice_id, role) - now = time.time() - # Single lock acquisition covers both the orchestrator-mode gate and - # the cooldown bookkeeping. - with self._lock: - # #3064 slice-5: in orchestrator mode session refresh happens at - # spawn (slice-4 worktree re-attach), not from heartbeat fan-out. - # Suppress the gateway call to prevent absent-sender alerts from - # stale heartbeat fan-out in orchestrator mode. - if self._orchestrator_mode: - return False - last = self._last_fan_out.get(key, 0.0) - if now - last < min_interval_seconds: - return False - self._last_fan_out[key] = now - return True - - def set_orchestrator_mode(self, enabled: bool = True) -> None: - """Enable or disable orchestrator-mode suppression of session fan-out. - - In orchestrator mode (``enabled=True``) the gateway-session refresh - side-effect is suppressed — the heartbeat route still validates, - dedups, and rate-limits, but ``should_fan_out_gateway_session`` - returns ``False``. Session refresh happens at spawn (slice-4 worktree - re-attach) not from heartbeat fan-out. Pod-mode (``enabled=False``) - restores today's fan-out behavior. Thread-safe. - """ - with self._lock: - self._orchestrator_mode = enabled - def clear(self, pipeline_id: str) -> None: """Drop all state for a pipeline (on phase transition). - Sweeps every ``(pipeline_id, *, *)`` key across all three + Sweeps every ``(pipeline_id, *, *)`` key across both per-(pipeline, slice, role) maps so a phase transition resets - rate budgets, dedup memory, and fan-out cooldowns for both - pipeline-level and slice-scoped agents in one shot. + rate budgets and dedup memory for both pipeline-level and + slice-scoped agents in one shot. """ with self._lock: for key in list(self._windows): @@ -215,9 +145,6 @@ def clear(self, pipeline_id: str) -> None: for key in list(self._last_state): if key[0] == pipeline_id: del self._last_state[key] - for key in list(self._last_fan_out): - if key[0] == pipeline_id: - del self._last_fan_out[key] _coordinator: HeartbeatCoordinator | None = None diff --git a/orchestrator/kubernetes_spawner.py b/orchestrator/kubernetes_spawner.py index c3001749e7..005680d431 100644 --- a/orchestrator/kubernetes_spawner.py +++ b/orchestrator/kubernetes_spawner.py @@ -216,12 +216,12 @@ def _resolve_wait_producer_allowlist(phase: str | None, role: str, repo: str | N LABEL_EVENT_DEDUPE = "egg.event.dedupe-key" LABEL_EVENT_ACTION = "egg.event.action" -# Env keys read by the consensus wrapper's one-shot arm -# (``consensus_wrapper.py``). ``EGG_EVENT_LOOP_OWNER=orchestrator`` + -# ``EGG_EVENT_ACTION`` engage the arm; ``EGG_EVENT_DEDUPE_KEY`` is the -# stale-event backstop / reconciliation handle. None are in -# ``_PROTECTED_ENV_KEYS`` — the one-shot entry is their only writer. -ENV_EVENT_LOOP_OWNER = "EGG_EVENT_LOOP_OWNER" +# Env keys read by the consensus wrapper's one-shot event handler +# (``consensus_wrapper.py``). ``EGG_EVENT_ACTION`` engages the handler; +# ``EGG_EVENT_DEDUPE_KEY`` is the stale-event backstop / reconciliation +# handle. None are in ``_PROTECTED_ENV_KEYS`` — the one-shot spawn is +# their only writer. (The ``EGG_EVENT_LOOP_OWNER`` ownership flag was +# retired in #3164 — the orchestrator always owns the event loop.) ENV_EVENT_ACTION = "EGG_EVENT_ACTION" ENV_EVENT_DEDUPE_KEY = "EGG_EVENT_DEDUPE_KEY" ENV_EVENT_PAYLOAD_REFS = "EGG_EVENT_PAYLOAD_REFS" @@ -1934,10 +1934,10 @@ def spawn_event_job( """Spawn (or adopt) a one-shot Job for a single BRC event (#3064). The Job's env carries the full event identity so the consensus - wrapper's one-shot arm engages (``EGG_EVENT_LOOP_OWNER=orchestrator`` - + ``EGG_EVENT_ACTION`` ∈ ``propose|ack|nack`` + ``EGG_EVENT_DEDUPE_KEY``) - and the dedupe key rides as a Job *label* — the reconciliation handle - the event loop rebuilds its live set from on restart. + wrapper's one-shot event handler engages (``EGG_EVENT_ACTION`` ∈ + ``propose|ack|nack`` + ``EGG_EVENT_DEDUPE_KEY``) and the dedupe key + rides as a Job *label* — the reconciliation handle the event loop + rebuilds its live set from on restart. **Adoption**: requesting a spawn for an already-live dedupe key returns ``None`` (the existing Job is adopted) rather than creating a @@ -2042,7 +2042,6 @@ def spawn_event_job( ) event_env: dict[str, str] = { - ENV_EVENT_LOOP_OWNER: "orchestrator", ENV_EVENT_ACTION: action, ENV_EVENT_DEDUPE_KEY: dedupe_key, } diff --git a/orchestrator/models.py b/orchestrator/models.py index 6fe3505704..cb8fcc7f3c 100644 --- a/orchestrator/models.py +++ b/orchestrator/models.py @@ -604,13 +604,15 @@ class PipelineConfig(BaseModel): ge=1, description=( "Per-pipeline cap on how many implement-phase slices run concurrently " - "in a slice-DAG wave. Set at pipeline creation. Each slice spawns ~8 " - "agent containers, so this is the primary host-load control for the " - "implement phase. When None (the default), the orchestrator falls back " - "to the EGG_ORCH_MAX_PARALLEL_SLICES env var, whose default is 1 — i.e. " - "a single slice runs at once unless explicitly raised here or via the " - "env var. The process-wide EGG_ORCH_GLOBAL_MAX_PARALLEL_SLICES cap " - "still applies across all pipelines." + "in a slice-DAG wave. Set at pipeline creation. Under on-demand " + "spawning (#3164) an in-flight slice costs at most one short-lived " + "one-shot pod per BRC event rather than a resident ~8-container " + "cohort, so this remains the primary implement-phase host-load knob " + "but at a far lower per-slice cost. When None (the default), the " + "orchestrator falls back to the EGG_ORCH_MAX_PARALLEL_SLICES env var, " + "whose default is 4. The process-wide " + "EGG_ORCH_GLOBAL_MAX_PARALLEL_SLICES cap still applies across all " + "pipelines." ), ) message_poll_hint_seconds: int = Field( diff --git a/orchestrator/routes/consensus.py b/orchestrator/routes/consensus.py index f1e27a5b8f..d98c813c14 100644 --- a/orchestrator/routes/consensus.py +++ b/orchestrator/routes/consensus.py @@ -618,10 +618,8 @@ def _build_iteration_feedback( ) -> dict[str, Any] | None: """Build the ``iteration_feedback`` block for a BRC event (#3231). - Under ``EGG_EVENT_LOOP_OWNER=orchestrator`` the re-spawned agent's - prompt is composed by ``orchestrator/routes/event_prompt.py`` (not the - in-pod ``_build_phase_iteration_context`` path that already carries - #2795's iteration context). Without this block: + The re-spawned agent's prompt is composed by + ``orchestrator/routes/event_prompt.py``. Without this block: * the **producer** (``propose`` arm) re-reads its own prior on-disk draft and re-proposes it byte-for-byte — the operator's @@ -809,9 +807,8 @@ def handle_next_action(pipeline_id: str) -> tuple[Response, int]: # Per-iteration operator kickback (#3231): thread the operator's # ``request_changes`` / ``change_approach`` feedback onto the - # event_payload so the re-spawned agent sees it under - # ``EGG_EVENT_LOOP_OWNER=orchestrator`` (its prompt is composed from - # this payload). Two arms carry it: + # event_payload so the re-spawned agent sees it (its prompt is + # composed from this payload). Two arms carry it: # * ``propose`` — the kicked-back producer must address (or rebut) # the directive before re-proposing, else it re-reads its own # prior draft and re-proposes it unchanged (the #1283 / #1915 diff --git a/orchestrator/routes/event_prompt.py b/orchestrator/routes/event_prompt.py index 2b0df7a80b..62639166c9 100644 --- a/orchestrator/routes/event_prompt.py +++ b/orchestrator/routes/event_prompt.py @@ -475,9 +475,7 @@ def _directive_meta_tag(directive: dict[str, Any]) -> str: def _render_iteration_feedback_section(iteration_feedback: dict[str, Any] | None) -> str: """Render the per-iteration operator kickback as a pushed section (#3231). - Under ``EGG_EVENT_LOOP_OWNER=orchestrator`` the re-spawned producer's - prompt is composed here (not via the in-pod ``_build_phase_iteration_context`` - path that already carries #2795's iteration context). Without this + The re-spawned producer's prompt is composed here. Without this section the producer re-reads its own prior on-disk draft and re-proposes it byte-for-byte — the operator's ``request_changes`` / ``change_approach`` silently no-ops (the #1283 / #1915 fake-cycle @@ -823,11 +821,12 @@ def compose_event_prompt( "Handle THIS single event per your role contract. Reuse the " "durable BRC memory below to keep your verdict consistent " "across one-shot invocations. When you have acted (proposed, " - "ACKed, NACKed, or confirmed), exit naturally — the wrapper " - "polls ``egg-orch brc next-action`` and re-invokes you with " - "the next actionable event. Do NOT block on " - "``egg-orch message wait-loop`` yourself: the wrapper owns " - "the wait and the heartbeat (#2908 slice-2).", + "ACKed, NACKed, or confirmed), exit naturally — the " + "orchestrator derives ``egg-orch brc next-action`` in-process " + "and re-spawns you one-shot with the next actionable event. " + "Do NOT block on ``egg-orch message wait-loop`` yourself: the " + "orchestrator owns the wait and spawns you one-shot per event " + "(#3164).", "", ] ) @@ -1633,8 +1632,7 @@ def _cli(argv: list[str] | None = None) -> int: # ``next-action`` route attaches ``iteration_feedback`` onto the # ``propose`` event_payload (sourced from # ``PhaseExecution.operator_directives`` / ``iteration_history``, - # #2795) so the re-spawned producer under - # ``EGG_EVENT_LOOP_OWNER=orchestrator`` addresses the operator's + # #2795) so the re-spawned producer addresses the operator's # ``request_changes`` / ``change_approach`` before re-proposing # instead of re-reading its own prior draft and re-proposing it # unchanged. Read straight off the event payload the wrapper pipes diff --git a/orchestrator/routes/messages.py b/orchestrator/routes/messages.py index 1abfa5568a..3cc35107db 100644 --- a/orchestrator/routes/messages.py +++ b/orchestrator/routes/messages.py @@ -138,17 +138,6 @@ def _track_long_poll_end() -> None: # Rate-limit (20/min per slice+role; #2471) still applies. _DEDUP_EXEMPT_HEARTBEAT_STATES: frozenset[str] = frozenset({"WAITING_FOR_EVENT"}) -# Minimum seconds between gateway-session fan-outs per (pipeline_id, -# slice_id, role) (#2076 NB2, slice-scoped per #2471). The dedup -# early-return path bypasses the per-(slice, role) heartbeat rate limit -# by design (#1897 NB1: dedup'd heartbeats are no-ops and must not -# consume rate budget), so without a separate cap a misbehaving agent -# hot-looping with identical state could amplify into the gateway at -# the agent's emission rate. The gateway's idle window -# is 60 minutes, so fanning out every 30 s is far more than enough to -# keep the session alive; the cap exists purely to bound amplification. -_GATEWAY_FANOUT_MIN_INTERVAL_SECONDS: float = 30.0 - messages_bp = Blueprint("messages", __name__, url_prefix="/api/v1/pipelines") @@ -867,20 +856,15 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]: # ``_DEDUP_EXEMPT_HEARTBEAT_STATES`` skip this check; see the # constant's docstring for the rationale. # - # Note: the gateway-session fan-out below runs *after* dedup but - # *before* rate-limit. Dedup'd heartbeats still fan out so an agent - # stuck in a single state (e.g. ``WORKING`` through a slow - # ``make test``) keeps its gateway session alive even when its BRC - # state hasn't changed. Rate-limited heartbeats do not fan out: by - # definition the agent already got plenty of refreshes in the last - # minute, and a hot-looping agent shouldn't amplify into the - # gateway. ``_refresh_gateway_session`` itself applies a separate - # per-role cooldown (#2076 NB2) to bound dedup-path amplification - # without consuming rate budget. + # #3164: the gateway-session fan-out that used to run here was + # retired with the in-pod wait arm. Under the orchestrator-owned + # event loop a pod is one-shot per BRC event, so its gateway session + # is refreshed at spawn time (worktree re-attach) and never needs a + # heartbeat-driven keep-alive — the resident-pod long-session model + # the fan-out existed for is gone. if state not in _DEDUP_EXEMPT_HEARTBEAT_STATES and coordinator.is_duplicate( pipeline_id, slice_id, from_role, state, waiting_on ): - _refresh_gateway_session(pipeline_id, from_role, slice_id) return _make_success( "HEARTBEAT deduped (unchanged state)", data={"deduped": True}, @@ -912,13 +896,11 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]: resp.headers["Retry-After"] = str(retry_after) return resp, 429 - # Refresh the agent's gateway session liveness (#2068). Runs after - # dedup and rate-limit gates: every accepted-or-deduped heartbeat - # fans out (dedup'd path above), but rate-limited ones do not. - # Best-effort: the gateway may be unreachable (tests, dev runs - # without a gateway) and a missing session is a 404; never fail the - # heartbeat on this path. - _refresh_gateway_session(pipeline_id, from_role, slice_id) + # #3164: the gateway-session fan-out that used to run here was retired + # with the in-pod wait arm. Under the orchestrator-owned event loop a + # pod is one-shot per BRC event, so its gateway session is refreshed at + # spawn time (worktree re-attach) and never needs a heartbeat-driven + # keep-alive. # Emit as a normal HEARTBEAT message on the bus so downstream # consumers (HealthMonitor, overseer, UI) see it. @@ -966,81 +948,6 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]: ) -def _refresh_gateway_session(pipeline_id: str, from_role: str, slice_id: str | None = None) -> None: - """Best-effort POST to the gateway so the BRC heartbeat counts as session liveness. - - Container-id normalization: k8s names are RFC-1123 labels (no - underscores), so ``kubernetes_spawner.JOB_NAME_FORMAT`` is filled - with ``agent_role.value.replace("_", "-")``. ``from_role`` arrives - from ``EGG_AGENT_ROLE`` which is the underscore form, so we mirror - the same normalization here — otherwise roles like - ``reviewer_refine`` build a container_id that never matches the - registered session and the gateway returns 404. See - ``orchestrator/kubernetes_spawner.py:370-375`` for the reference - pattern. - - Slice scope (#2451): slice-scoped agents register sessions under - ``egg-agent-{pid}-{slice_id}-{role}`` (``JOB_NAME_FORMAT_SLICE``); - pipeline-level agents register under ``egg-agent-{pid}-{role}``. - When ``slice_id`` is supplied (forwarded from the heartbeat body - via ``EGG_SLICE_ID``) the slice segment is embedded so the lookup - matches; without it every slice-scoped agent's heartbeat fan-out - would silently 404. The throttle key is also slice-aware so a - sibling slice's fan-out does not suppress this slice's refresh. - - Trust model: ``from_role`` is taken at face value from the request - body and is **not** correlated against the calling container's - session. This matches the existing message-bus trust model — any - agent in any container can already post messages claiming to be - another role — but with this fan-out a misbehaving agent can keep - a sibling's gateway session alive past the idle timeout. Tracked - as a follow-up; spoofing here doesn't grant any new capability, - only extends an existing session's lifetime. - - Throttle: per-role cooldown via - ``HeartbeatCoordinator.should_fan_out_gateway_session`` (#2076 NB2) - bounds amplification on the dedup early-return path, which bypasses - the heartbeat rate limiter by design. The cooldown is well below - the gateway's 60-minute idle window so it does not risk session - expiry under any realistic heartbeat cadence. - """ - coordinator = get_heartbeat_coordinator() - # The coordinator's throttle key is now ``(pipeline_id, slice_id, - # role)`` (#2471) so concurrent slices that share a role (e.g. two - # reviewer-code agents in different slices of the same wave) do not - # suppress each other's fan-outs. Pass ``slice_id`` directly — no - # synthetic role-string composition needed. - if not coordinator.should_fan_out_gateway_session( - pipeline_id, slice_id, from_role, _GATEWAY_FANOUT_MIN_INTERVAL_SECONDS - ): - return - try: - try: - from gateway_client import get_gateway_client - except ImportError: # pragma: no cover - from ..gateway_client import ( - get_gateway_client, # type: ignore[no-redef,import-not-found] - ) - - # Mirror kubernetes_spawner.JOB_NAME_FORMAT's role normalization - # — k8s labels disallow underscores, so the registered - # container_id uses hyphens. - normalized_role = from_role.replace("_", "-") - if slice_id: - container_id = f"egg-agent-{pipeline_id}-{slice_id}-{normalized_role}" - else: - container_id = f"egg-agent-{pipeline_id}-{normalized_role}" - get_gateway_client().heartbeat_session_by_container(container_id) - except Exception as exc: # pragma: no cover - logging only - logger.warning( - "Gateway session heartbeat fan-out failed", - pipeline_id=pipeline_id, - from_role=from_role, - slice_id=slice_id, - error=str(exc), - ) - - @messages_bp.route("//messages/status", methods=["GET"]) def message_status(pipeline_id: str) -> tuple[Response, int]: """Get message bus status for a pipeline.""" diff --git a/orchestrator/routes/pipelines.py b/orchestrator/routes/pipelines.py index 5b27c22cce..0a408b22dd 100644 --- a/orchestrator/routes/pipelines.py +++ b/orchestrator/routes/pipelines.py @@ -290,6 +290,7 @@ def get_repo_checks(repo: str) -> list[dict[str, str]]: # type: ignore[misc] _rebase_with_agent_output_autoresolve, ) from ..kubernetes_client import ( + LABEL_AGENT_ROLE, LABEL_PIPELINE_ID, LABEL_SLICE_ID, JobOperationError, @@ -345,6 +346,7 @@ def get_repo_checks(repo: str) -> list[dict[str, str]]: # type: ignore[misc] _rebase_with_agent_output_autoresolve, ) from kubernetes_client import ( # type: ignore + LABEL_AGENT_ROLE, LABEL_PIPELINE_ID, LABEL_SLICE_ID, JobOperationError, @@ -1229,7 +1231,7 @@ def _guard_live_pods_or_force( return None -from routes import get_repo_path, resolve_worktree_repo_path # noqa: E402 — shared helpers +from routes import get_repo_path # noqa: E402 — shared helpers try: from gateway_client import get_gateway_client @@ -2646,10 +2648,26 @@ def delete_pipeline(pipeline_id: str) -> tuple[Response, int]: @pipelines_bp.route("//agents//restart", methods=["POST"]) @require_lifecycle_secret def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: - """Restart a single agent in a pipeline. - - Stops the existing container, resets its consensus state, and respawns - it with the same configuration. The agent's per-agent worktree is + """Restart a single agent in a pipeline (orchestrator-native). + + After #3164 the orchestrator unconditionally owns the BRC event + loop: agent work runs as one-shot Jobs spawned per actionable + event by the event loop, and the in-pod wait arm is gone. A + resident pod spawned here without ``EGG_EVENT_ACTION`` would + immediately log FATAL and ``exit 64``, so ``restart_agent`` no + longer spawns anything itself. Instead it: + + 1. Best-effort deletes the role's live one-shot Job(s) (to kill a + stuck pod). One-shot Jobs carry an event-discriminator suffix + in their name, so they are found by label + (``LABEL_PIPELINE_ID`` + ``LABEL_AGENT_ROLE`` [+ ``LABEL_SLICE_ID`` + when slice-scoped]), not by name. + 2. Resets the role's consensus state and health-monitor anchor. + 3. Marks the agent record RUNNING with ``container_id = None``. + + The event loop (polling ~every 5s during the concurrent phase) then + spawns a fresh one-shot pod once the role's consensus state is + reset — that is the respawn. The agent's per-agent worktree is preserved so committed work is retained. URL params: @@ -2686,8 +2704,9 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: { "success": true, "data": { - "container_id": "abc123...", "agent_role": "coder", + "slice_id": "slice-2", + "respawn": "delegated to orchestrator event loop", "restart_count": 1 } } @@ -2812,31 +2831,9 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: # contract not yet populated, filesystem error) so we don't # regress legitimate restarts on the existing pipeline-level path. # - # The contract is also consulted to resolve the slice's parent - # edge (#2439) so the spawner's ``base_branch`` matches the - # parent slice's integration branch on a worktree-absent restart. - parent_slice_id: str | None = None - # ``parent_branch_recorded`` captures ``Slice.parent_branch_at_creation`` - # — the literal branch the parent slice's integration branch was forked - # off of when its worktree was provisioned (#2137 TASK-4-2). Preferring - # the recorded value over reconstructing - # ``f"{_issue_branch}/{parent_slice_id}"`` is more robust: if a future - # qualifier-suffix or namespacing change lands in - # ``_run_one_slice_inner`` but not here, the reconstruction would - # silently drift while the recorded value would not (#2460 review). - # ``None`` for slices whose worktree has not been provisioned yet, in - # which case we fall through to reconstruction. - parent_branch_recorded: str | None = None - # ``parent_slice_complete`` is set when the parent slice has reached - # ``SliceStatus.COMPLETE`` per the contract — i.e. its PR has plausibly - # been merged. GitHub's standard branch-auto-cleanup deletes the head - # branch on merge, so the gateway's per-repo ``git fetch origin - # `` would wedge the restart on a missing-branch fetch - # error. When ``True``, we fall back to ``pipeline.base_branch`` rather - # than depend on a (likely deleted) parent ref — matching the - # contract-unloadable fall-through policy: prefer letting the restart - # proceed over over-strict gating (#2470). - parent_slice_complete: bool = False + # After #3164 ``restart_agent`` no longer spawns a worktree itself, + # so the slice's parent-edge / base-branch resolution that used to + # feed the spawn is gone. Only the existence check below remains. if slice_id is not None: if not pipeline.has_contract: return make_error_response( @@ -2886,9 +2883,6 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: error=str(exc), ) if contract is not None: - # Single-pass slice lookup: the existence check and the - # parent-edge read both need the same record, so do them - # together (#2460 review observation 4). slice_obj = next((s for s in contract.slices if s.id == slice_id), None) if slice_obj is None: return make_error_response( @@ -2900,39 +2894,17 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: "known_slices": sorted(s.id for s in contract.slices), }, ) - # Resolve the slice's parent edge (#2439). The forest - # constraint enforced by contract ingestion guarantees - # at most one DAG parent per slice, so taking the first - # dependency is sufficient. Root slices (no parent) - # leave ``parent_slice_id`` as ``None``. - if slice_obj.dependencies: - parent_slice_id = slice_obj.dependencies[0] - parent_branch_recorded = slice_obj.parent_branch_at_creation - # #2470: if the parent slice is complete, its PR has - # plausibly been merged and its branch deleted by - # GitHub auto-cleanup. Detect that here so the - # base-branch selection below can fall back to - # ``pipeline.base_branch`` rather than wedge the - # restart on a missing-branch fetch. - from egg_contracts.models import SliceStatus - - parent_obj = next( - (s for s in contract.slices if s.id == parent_slice_id), - None, - ) - if parent_obj is not None and parent_obj.status == SliceStatus.COMPLETE: - parent_slice_complete = True - # Restart the container via spawner spawner = _get_spawner() - # Gather spawn parameters from pipeline state current_phase = pipeline.current_phase.value phase_exec = pipeline.phases.get(current_phase) - # Early status update: transition FAILED/CANCELLED -> RUNNING before the - # slow container restart so that get_status returns "running" immediately, - # even if the MCP call times out (see #1594, #1725). + # Early status update: transition FAILED/CANCELLED -> RUNNING so that + # get_status returns "running" immediately and the event loop (which + # only acts on RUNNING pipelines) is unblocked to respawn (see #1594, + # #1725). With no resident spawn (#3164) there is no failure path that + # would revert this transition. if pipeline.status in (PipelineStatus.FAILED, PipelineStatus.CANCELLED): early_lock = get_pipeline_state_lock(pipeline_id) with early_lock: @@ -2945,333 +2917,71 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: pipeline.updated_at = datetime.now(UTC) store.update_pipeline(pipeline_id, pipeline.model_dump(mode="json")) - # Compute gateway mode from pipeline config (not hardcoded "public") - gateway_mode, _ = _compute_gateway_mode(pipeline) - - # Resolve the per-agent worktree path and the default branch once, - # so the pipeline-level / root-slice / parent-complete branches of - # the slice-aware fallback below can substitute a concrete branch - # name when ``pipeline.base_branch`` is ``None`` (auto-detect). - # Without this, the spawner's ``EGG_BASE_BRANCH`` env-injection guard - # (``if base_branch:``) skips the export, the BRC event-pump's - # per-producer ``git log {sha}..HEAD --not origin/`` delta - # falls back to ``origin/main`` in both consumers, and the diff - # errors out on every non-``main`` repo (#2967) — the exact failure - # mode the initial-spawn path was fixed for, surfacing on every - # health-monitor-triggered or operator-driven restart. The resolved - # value is also threaded into the prompt's ``base_branch=`` argument - # below so the prompt's diff base and the spawner's worktree base - # cannot diverge within the restart path. - # - # ``worktree_repo_path`` and the default-branch lookup can fail on a - # pipeline with no resolvable repo (e.g. ``pipeline.repo`` is ``None`` - # or the worktree was pruned). Preserve pre-#2967 restart availability - # by catching and falling back to the raw ``pipeline.base_branch`` — - # restart was unconditional before this resolution was added, so - # failing the resolution must not regress that. - _restart_env_path = os.environ.get("EGG_REPO_PATH", "/home/egg/repos") - _restart_base_path = Path(_restart_env_path) - _restart_repo_name = (pipeline.repo or "").split("/")[-1] - worktree_repo_path: Path | None - try: - worktree_repo_path = resolve_worktree_repo_path(_restart_base_path, _restart_repo_name) - except Exception as _wt_err: - logger.warning( - "Could not resolve worktree repo path for restart base-branch " - "resolution; falling back to raw pipeline.base_branch", - pipeline_id=pipeline_id, - agent_role=agent_role, - error=str(_wt_err), - ) - worktree_repo_path = None - _resolved_base_branch_for_restart: str | None = pipeline.base_branch - if not _resolved_base_branch_for_restart and worktree_repo_path is not None: - try: - _resolved_base_branch_for_restart = get_default_branch(worktree_repo_path) - except Exception: - _resolved_base_branch_for_restart = None - - # Slice-aware restart branch (#2428). When the restart targets a - # slice agent, the spawner must register the gateway session - # against the slice integration branch (``/``); the - # pipeline tip (``/work``) is the wrong target and every - # subsequent push from the restarted agent would be rejected by - # the gateway's branch allowlist. Mirror the slice scheduler's - # derivation in :func:`_run_implement_phase_slices` so a restarted - # slice agent lands on the same integration branch its peers are - # using. - # - # The spawner's ``base_branch`` is the ref the per-agent worktree - # is forked off of when the worktree must be (re)created (#2439). - # ``pipeline.branch`` (the integration tip) is the wrong target — - # if the worktree is absent at restart time, forking from the - # pipeline tip pulls sibling slices' commits into the rebuilt - # worktree. The right base depends on context: - # - Pipeline-level restart: ``_resolved_base_branch_for_restart`` - # (``pipeline.base_branch`` if explicit, else the worktree's - # ``get_default_branch`` so the spawner exports a concrete - # ``EGG_BASE_BRANCH`` on master-default repos — #2967 follow-up). - # - Slice restart with a parent in the contract's slice forest: - # the parent slice's integration branch - # (``/``), mirroring ``parent_branch`` - # in :func:`_run_one_slice_inner`. - # - Root-slice restart, or slice restart when the contract - # can't be loaded: ``_resolved_base_branch_for_restart`` (fallback, - # same resolution as the pipeline-level case). - # - # Note: this intentionally diverges from the initial-spawn path at - # :func:`_run_concurrent_phase` for the parent-slice cases — - # ``_run_concurrent_phase`` now passes the resolved - # ``_resolved_base_branch`` (the same shape this path uses for - # pipeline-level / root-slice / parent-complete restarts), but #2439 - # specifically asks for the parent-slice fork on the *restart* path - # so a worktree-absent restart of a child slice rebuilds atop its - # parent slice rather than re-forking from - # ``pipeline.base_branch`` and losing the parent's commits. Don't - # "fix" this asymmetry by aligning the two paths without first - # re-reading #2439. + # #3164: ``restart_agent`` no longer spawns a resident pod. The + # orchestrator event loop owns the BRC respawn — once the role's + # consensus state is reset (below), it spawns a fresh one-shot pod + # within one ~5s poll. Here we only (1) kill any live one-shot Job + # for the role so a stuck pod is torn down, then (2) reset consensus + # + health so the event loop reschedules. + + # Delete the role's live one-shot Job(s), best-effort. One-shot + # event Jobs carry an event-discriminator SUFFIX in their name (one + # Job per actionable BRC event), so they can't be addressed by a + # deterministic name — find them by LABEL. Match on pipeline + + # role (and slice when scoped). Zero matches is fine (the role may + # have already exited cleanly); the event loop will respawn either + # way once consensus is reset. Wrap broadly so a k8s/list failure + # never fails the restart. + job_labels = { + LABEL_PIPELINE_ID: pipeline_id, + # The role label value is the underscore form (e.g. + # ``reviewer_code``), which is exactly ``agent_role`` / ``role.value``. + LABEL_AGENT_ROLE: role.value, + } if slice_id is not None: - _pipeline_branch = pipeline.branch or ( - f"egg/issue-{pipeline.issue_number}/work" - if pipeline.issue_number is not None - else f"egg/{pipeline_id}/work" - ) - _issue_branch = _slice_namespace_root(_pipeline_branch) - # Defense-in-depth: re-validate the slice id shape before - # embedding it in a git ref. ``extract_slice_id`` (above) - # already enforces ``^slice-[0-9]+$`` on the request payload, - # but the helper is part of the gateway-facing surface — a - # future caller that forgets upstream validation must not be - # able to smuggle path separators or shell metacharacters in - # via this seam. Mirrors the check at - # ``concurrent_executor.get_worktree_branch`` so both spawn - # entrypoints share the same canonical pattern. - if not SLICE_ID_PATTERN.fullmatch(slice_id): - raise ValueError( - f"slice_id={slice_id!r} does not match the canonical shape ``slice-``" - ) - agent_branch = f"{_issue_branch}/{slice_id}" - if parent_slice_id is not None and parent_slice_complete: - # #2470: parent slice's PR has plausibly been merged and its - # branch deleted by GitHub auto-cleanup. Falling back to - # ``_resolved_base_branch_for_restart`` is safe: ``complete`` - # means the parent's commits have been integrated upstream - # (either via PR merge or via the cascade), and prefer - # letting the restart proceed over wedging on a missing- - # branch fetch. Resolved (rather than raw - # ``pipeline.base_branch``) so the spawner exports - # ``EGG_BASE_BRANCH`` on auto-detect repos (#2967 follow-up). - base_branch_for_restart = _resolved_base_branch_for_restart - elif parent_slice_id is not None: - if parent_branch_recorded: - # Prefer the literal branch the parent slice was - # provisioned against (#2460 review observation 2). - # Set by ``_run_one_slice_inner`` at slice creation - # time; per the docstring on - # ``Slice.parent_branch_at_creation``, it is *the* - # recorded fact about how the slice was provisioned. - # We trust our own writer here — no extra ref-shape - # validation, just the gateway's per-repo ``git - # fetch`` will surface a malformed value. - base_branch_for_restart = parent_branch_recorded - else: - # Fallback: contract has the dependency edge but no - # provisioning record. Reconstruct from the slice - # namespace root and the parent's id. - # - # Defense-in-depth: ``parent_slice_id`` came from the - # contract loader (whose ``Slice.id`` regex permits - # both canonical ``slice-`` and legacy - # ``phase-``), so accept either shape before - # embedding into a git ref. Anything outside that - # envelope is a corrupt-contract smell — fail loudly - # rather than synthesising a malformed ref the - # gateway would reject anyway. - if not _SLICE_OR_PHASE_ID_PATTERN.fullmatch(parent_slice_id): - raise ValueError( - f"parent_slice_id={parent_slice_id!r} does not match " - f"the canonical shape ``slice-`` (or legacy ``phase-``)" - ) - base_branch_for_restart = f"{_issue_branch}/{parent_slice_id}" - else: - # Root slice (no parent edge): same shape as the pipeline-level - # case below — resolved so a ``None`` ``pipeline.base_branch`` - # still reaches the spawner as a concrete branch name - # (#2967 follow-up). - base_branch_for_restart = _resolved_base_branch_for_restart - else: - agent_branch = pipeline.branch - # Pipeline-level restart: resolved so the spawner exports - # ``EGG_BASE_BRANCH`` and the BRC delta works on auto-detect repos - # (master-default), matching the initial-spawn path's behavior at - # ``_run_concurrent_phase`` (#2967 follow-up). - base_branch_for_restart = _resolved_base_branch_for_restart - - # Reconstruct command and extra_env for concurrent agents. - # In concurrent mode, agents need a consensus-wrapped prompt command - # and role-specific environment variables to function properly. - command = None - extra_env: dict[str, str] = {} - - # Per-agent model resolution for the restart path (#2769 task-2-5). - # The default Anthropic decision keeps every restart kwarg byte- - # identical to the pre-#2769 wire shape; non-default decisions wire - # the right ``--model`` into the consensus wrapper and the right - # upstream/upstream_model into the gateway session. + job_labels[LABEL_SLICE_ID] = slice_id try: - try: - from agent_model_resolution import ( - UPSTREAM_ANTHROPIC, - resolve_agent_model, - ) - except ImportError: - from ..agent_model_resolution import ( # type: ignore[import-not-found, no-redef] - UPSTREAM_ANTHROPIC, - resolve_agent_model, - ) - _model_decision = resolve_agent_model( - role=role, - pipeline_config=pipeline.config, - repo=pipeline.repo, - ) - except Exception as resolve_err: - # Resolution is pure data over already-validated inputs; logging - # and falling back to the built-in opus default preserves restart - # availability even if a future regression breaks the resolver. - logger.warning( - "Failed to resolve per-agent model decision for restart, " - "falling back to built-in opus default", - pipeline_id=pipeline_id, - agent_role=agent_role, - error=str(resolve_err), - ) - try: - from agent_model_resolution import UPSTREAM_ANTHROPIC, classify_model - except ImportError: - from ..agent_model_resolution import ( # type: ignore[import-not-found, no-redef] - UPSTREAM_ANTHROPIC, - classify_model, - ) - - _model_decision = classify_model("opus") - - try: - try: - from concurrent_executor import ConcurrentPhaseExecutor, is_concurrent_execution - except ImportError: - from ..concurrent_executor import ConcurrentPhaseExecutor, is_concurrent_execution - - if is_concurrent_execution(pipeline, phase=current_phase): - # Reconstruct extra_env via ConcurrentPhaseExecutor - executor = ConcurrentPhaseExecutor(pipeline, spawn_fn=lambda **kw: None) # type: ignore[arg-type] - extra_env = executor.get_agent_env(role) - - # Reconstruct the agent prompt and wrap it for consensus. - # ``worktree_repo_path`` and ``_resolved_base_branch_for_restart`` - # were already computed above the slice-aware fallback so the - # prompt's diff base, the spawner's ``base_branch`` argument - # (and thus the exported ``EGG_BASE_BRANCH``), and the worktree - # base cannot diverge within this restart path (#2967 follow-up). - # ``worktree_repo_path`` is ``None`` only when the hoisted - # resolution failed (logged above); skip prompt reconstruction - # in that case rather than passing ``"None"`` as ``repo_path``. + live_jobs = spawner.k8s.list_containers(labels=job_labels) + removed_jobs = 0 + for job in live_jobs: try: - if worktree_repo_path is None: - raise RuntimeError( - "worktree_repo_path unavailable; skipping prompt reconstruction" - ) - prompt_text = _build_agent_prompt( - role_value=agent_role, - phase=current_phase, - pipeline_id=pipeline_id, - pipeline_mode=pipeline.mode.value if pipeline.mode else "issue", - prompt=pipeline.prompt, - issue_number=pipeline.issue_number, - repo=pipeline.repo, - branch=pipeline.branch, - base_branch=_resolved_base_branch_for_restart, - repo_path=str(worktree_repo_path), - concurrent=True, - network_mode=gateway_mode, - ) - if prompt_text: - from consensus_wrapper import build_consensus_wrapped_command - - command = build_consensus_wrapped_command( - prompt_text, - model=_model_decision.claude_code_alias, - effort=_model_decision.effort, - ) - except Exception as prompt_err: + # Mirror the cleanup call sites: prefer the explicit + # ``job_name`` (already Job-prefixed), fall back to the + # container id which ``remove_agent_job`` -> ``remove_container`` + # resolves to a Job name. + spawner.remove_agent_job(job.job_name or job.container_id, force=True) + removed_jobs += 1 + except Exception as job_err: # noqa: BLE001 - best-effort teardown logger.warning( - "Failed to reconstruct agent prompt for restart " - "(agent will start without a prompt command)", + "Failed to delete live one-shot Job during restart (best-effort)", pipeline_id=pipeline_id, agent_role=agent_role, - error=str(prompt_err), + slice_id=slice_id, + job_name=getattr(job, "job_name", None), + error=str(job_err), ) - except ImportError: - logger.debug("Concurrent executor not available for restart prompt reconstruction") - except Exception as e: - logger.warning( - "Failed to reconstruct concurrent env for restart", + logger.info( + "restart_agent: deleted live one-shot Job(s) for role", pipeline_id=pipeline_id, agent_role=agent_role, - error=str(e), + slice_id=slice_id, + removed=removed_jobs, ) - - # Forward upstream/upstream_model only on non-default decisions so a - # restart on the Claude default keeps the spawner kwargs byte- - # identical to today's pre-#2769 shape (regression guard). - restart_upstream_kwargs: dict[str, str | None] = {} - if _model_decision.upstream != UPSTREAM_ANTHROPIC or _model_decision.upstream_model is not None: - restart_upstream_kwargs["upstream"] = _model_decision.upstream - restart_upstream_kwargs["upstream_model"] = _model_decision.upstream_model - - # Merge the model-decision env vars so the restarted agent matches the - # initial spawn. On the LiteLLM path these include the - # ANTHROPIC_CUSTOM_MODEL_OPTION vars (#2832) for custom-model registration; - # every route also picks up the context-guardrail caps (#3175), so this is - # non-empty on the Anthropic path too — restarted Anthropic agents inherit - # the same guardrails as the initial spawn (matches concurrent_executor). - extra_env = {**extra_env, **_model_decision.env_vars()} - - try: - spawned = spawner.restart_agent_container( + except Exception as list_err: # noqa: BLE001 - best-effort teardown + logger.warning( + "Failed to list live one-shot Jobs during restart (best-effort)", pipeline_id=pipeline_id, - agent_role=role, - issue_number=pipeline.issue_number, - mode=gateway_mode, - extra_env=extra_env or None, - repos=[pipeline.repo] if pipeline.repo else None, - phase=current_phase, - command=command, - branch=agent_branch, - base_branch=base_branch_for_restart, - reason=reason, - spawn_max_retries=pipeline.config.spawn_max_retries, - spawn_retry_initial_backoff_seconds=pipeline.config.spawn_retry_initial_backoff_seconds, + agent_role=agent_role, slice_id=slice_id, - **restart_upstream_kwargs, - ) - except (ContainerSpawnError, KubernetesSpawnError) as e: - # Revert early status update — the agent is not actually running. - # Consensus state is intentionally NOT reset here so a failed spawn - # preserves the agent's prior consensus participation. - revert_lock = get_pipeline_state_lock(pipeline_id) - with revert_lock: - pipeline = store.load_pipeline(pipeline_id) - pipeline.status = PipelineStatus.FAILED - _phase_exec = pipeline.phases.get(current_phase) - if _phase_exec is not None: - _phase_exec.status = PipelineStatus.FAILED - pipeline.updated_at = datetime.now(UTC) - store.update_pipeline(pipeline_id, pipeline.model_dump(mode="json")) - return make_error_response(f"Failed to restart agent: {e}", status_code=500) - - # Spawn succeeded — now reset consensus state for this agent. - # If consensus reset fails, log a warning but don't fail the restart: - # the restarted agent will re-enter consensus on its own. + error=str(list_err), + ) + + # Reset consensus state for this agent so the event loop reschedules + # a fresh one-shot pod for it. If consensus reset fails, log a + # warning but don't fail the restart: the agent will re-enter + # consensus on its own. Slice-scoped restarts (#2410) target the + # per-slice tracker; the pipeline-level tracker has no record of the + # slice agent. # Slice-scoped restarts (#2410) target the per-slice tracker; the # pipeline-level tracker has no record of the slice agent. try: @@ -3323,34 +3033,29 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: error=str(e), ) - # Update pipeline state with new container/agent info + # Update pipeline state. No resident container is spawned (#3164) — + # the event loop will respawn a one-shot pod within one poll once the + # consensus reset above takes effect. We mark the agent RUNNING with + # ``container_id = None`` (the live pod is set by the event loop) and + # refresh ``started_at`` so the overseer's + # phase_minimum_working_window suppression on the + # ``agent-heartbeat-stall`` trigger anchors on the restart (#2084). lock = get_pipeline_state_lock(pipeline_id) with lock: pipeline = store.load_pipeline(pipeline_id) if phase_exec is not None: # Re-fetch from the freshly loaded pipeline (the outer check gates - # on "did the phase exist before the spawn?"). + # on "did the phase exist before the restart?"). fresh_phase_exec = pipeline.phases.get(current_phase) if fresh_phase_exec is not None: - # Add new container info - fresh_phase_exec.containers.append(spawned.container_info) - - # Update or add agent execution entry from models import AgentExecution # type: ignore - # Refresh ``started_at`` to the new container's spawn time so - # ``_get_concurrent_status`` reports an ``elapsed_seconds`` - # anchored on the live container. Without this the field - # carries the original spawn timestamp and the overseer's - # phase_minimum_working_window suppression on the - # ``agent-heartbeat-stall`` trigger is structurally dead on - # the ``restart_agent`` path (issue #2084). respawn_started_at = datetime.now(UTC) # Match on ``(role, slice_id)`` — without the slice tiebreaker # the first matching role wins, which on a multi-slice phase # mutates the wrong slice's record (#2422). ``slice_id`` is - # the route-level scope already plumbed into the spawner and - # consensus tracker above. + # the route-level scope already plumbed into the consensus + # tracker above. found = False for agent in fresh_phase_exec.agents: if not hasattr(agent, "role"): @@ -3362,51 +3067,63 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: continue if getattr(agent, "slice_id", None) != slice_id: continue - agent.container_id = spawned.container_info.container_id + agent.container_id = None agent.status = AgentExecutionStatus.RUNNING agent.started_at = respawn_started_at - agent.resolved_model = _model_decision.claude_code_alias found = True break if not found: fresh_phase_exec.agents.append( AgentExecution( role=role, - container_id=spawned.container_info.container_id, + container_id=None, status=AgentExecutionStatus.RUNNING, started_at=respawn_started_at, slice_id=slice_id, - resolved_model=_model_decision.claude_code_alias, ) ) pipeline.updated_at = datetime.now(UTC) store.update_pipeline(pipeline_id, pipeline.model_dump(mode="json")) - # Slice-scoped restarts (#2410) bumped the per-slice budget bucket + # Slice-scoped restarts (#2410) read the per-slice budget bucket # ``(pipeline_id, agent_role, slice_id)``; the pipeline-level # ``(pipeline_id, agent_role, None)`` bucket is untouched. Reading # without ``slice_id`` here would return the pipeline-level count - # (typically zero) and the audit log + JSON response below would - # misreport the operator's "you've burned N of M restarts" telemetry. - restart_count = spawner.get_restart_count(pipeline_id, agent_role, slice_id=slice_id) + # (typically zero) and the JSON response below would misreport the + # operator's "you've burned N of M restarts" telemetry. Best-effort: + # the count is informational, so a lookup failure must not fail the + # restart. + response_data: dict[str, object] = { + "agent_role": agent_role, + "slice_id": slice_id, + "respawn": "delegated to orchestrator event loop", + } + try: + response_data["restart_count"] = spawner.get_restart_count( + pipeline_id, agent_role, slice_id=slice_id + ) + except Exception as count_err: # noqa: BLE001 - informational only + logger.warning( + "Failed to read restart_count for restart response", + pipeline_id=pipeline_id, + agent_role=agent_role, + slice_id=slice_id, + error=str(count_err), + ) logger.info( - "Agent restarted", + "Agent restart requested (respawn delegated to event loop)", pipeline_id=pipeline_id, agent_role=agent_role, - container_id=spawned.container_info.container_id[:12], - restart_count=restart_count, + slice_id=slice_id, + restart_count=response_data.get("restart_count"), reason=reason, ) return make_success_response( f"Agent {agent_role} restarted", - data={ - "container_id": spawned.container_info.container_id, - "agent_role": agent_role, - "restart_count": restart_count, - }, + data=response_data, ) diff --git a/orchestrator/tests/golden/event_pump_wrapper.sh.golden b/orchestrator/tests/golden/event_pump_wrapper.sh.golden new file mode 100644 index 0000000000..8aea5d14cf --- /dev/null +++ b/orchestrator/tests/golden/event_pump_wrapper.sh.golden @@ -0,0 +1,384 @@ +#!/bin/bash +set -uo pipefail + +# One-shot event wrapper (#3164). The orchestrator spawns this wrapper +# once per actionable BRC event (``EGG_EVENT_ACTION`` injected); the +# helper functions below are shared by the one-shot event handler +# appended at the end. No blocking wait-loop and no background heartbeat: +# the orchestrator owns all waiting. + +cw_log() { + echo "[event-pump] $*" >&2 +} + +# Emit one heartbeat. The CLI's ``message heartbeat`` handler auto- +# attaches ``slice_id`` from ``$EGG_SLICE_ID`` via +# ``_maybe_attach_slice_id`` in +# ``sandbox/egg_agent_tools/handlers/_gateway.py``. The one-shot event +# handler emits a single foreground ping (no background emitter) so the +# slice-5 silent-mid-event tripwire keys on heartbeats from an active +# one-shot pod. We also echo the slice tag in the ``--body`` so a +# snapshot test can grep for the slice_id propagation without +# intercepting the HTTP POST. +emit_heartbeat() { + local state="$1" + local body_text="$2" + local slice_tag="${EGG_SLICE_ID:-none}" + timeout 5 egg-orch message heartbeat \ + --state "$state" \ + --body "$body_text (slice=$slice_tag)" \ + >/dev/null 2>&1 || true +} + +# Ask the orchestrator route what to do next. Returns a JSON document +# with ``action`` ('wait'|'propose'|'ack'|'nack'|'confirm'|'complete') +# and an optional ``event_payload``. The one-shot handler uses this once +# as a stale-event freshness re-check before invoking the agent. +# +# HTTP 409 from the route (409 stale_version, 409 aggregated-NACK +# barrier) is a signal, not a crash: the CLI surfaces it as a non-zero +# exit code with an empty/invalid JSON. We fall back to +# ``{"action":"wait"}`` and propagate the original ``rc`` so the +# one-shot handler can tell an inconclusive re-check (exit 75) from a +# positively-derived action. +fetch_next_action() { + local out rc + out=$(egg-orch brc next-action --role "${EGG_AGENT_ROLE:-unknown}" --json 2>/dev/null) + rc=$? + if [ "$rc" -eq 0 ] && [ -n "$out" ]; then + echo "$out" + return 0 + fi + # The CLI returns non-zero for any of: 409 stale_version, 409 + # aggregated-NACK barrier, 5xx, transport failure. Log it so + # operators reading wrapper logs can distinguish "orchestrator + # returned 409" (benign, expected) from "transport unreachable" + # (worth checking). We always echo the fallback JSON so the parser + # doesn't crash, and propagate the original ``rc`` as the function's + # exit code (the caller reads ``$?`` after the ``$(...)`` capture). + cw_log "brc next-action returned rc=$rc / empty body; falling back to {\"action\":\"wait\"}." + echo '{"action":"wait"}' + return "$rc" +} + +# Extract a top-level scalar field from a next-action JSON document. +# Nested objects (event_payload) are re-serialised as JSON so the +# bash caller can pass them downstream verbatim. +next_action_field() { + local action_json="$1" + local field="$2" + echo "$action_json" | python3 -c " +import sys, json +field = sys.argv[1] +try: + d = json.load(sys.stdin) +except Exception: + print(''); sys.exit(0) +v = d.get(field) +if isinstance(v, (dict, list)): + print(json.dumps(v)) +elif v is None: + print('') +else: + print(v) +" "$field" 2>/dev/null || echo "" +} + +# Invoke the agent one-shot with the per-event prompt. Slice-3 +# (TASK-3-1 / TASK-3-2) replaces the slice-2 stub with the full +# ``compose_event_prompt`` payload: memory excerpt (when +# ``EGG_BRC_MEMORY=full``) + per-producer +# ``git log {sha}..HEAD --not origin/{base} -p`` delta + NACK +# payload + the contract's ``task_description`` (#3123, read from the +# worktree contract file via the pod-inherited ``EGG_PIPELINE_ID`` / +# ``EGG_ISSUE_NUMBER``). The composer lives at +# ``/opt/egg-runtime/orchestrator/routes/event_prompt.py`` -- the +# wrapper invokes its ``if __name__ == '__main__'`` CLI directly so the +# heavy ``orchestrator.routes`` package ``__init__.py`` (Flask import) +# is bypassed. ``EGG_EVENT_PROMPT_SCRIPT`` overrides the path for tests. +# +# When the composer fails (script missing, malformed memory file, git +# log subprocess crash) we fall back to the slice-2 minimal stub so the +# event-pump keeps running rather than failing the agent invocation. +# This is symmetric with the rest of the wrapper's "block, alert, +# continue" stance under the idle-budget safety net. +invoke_agent_for_event() { + local action="$1" + local event_payload="$2" + local role="${EGG_AGENT_ROLE:-unknown}" + local slice="${EGG_SLICE_ID:-none}" + local base_branch="${EGG_BASE_BRANCH:-main}" + local script_path="${EGG_EVENT_PROMPT_SCRIPT:-/opt/egg-runtime/orchestrator/routes/event_prompt.py}" + local prompt prompt_rc=1 + + if [ -r "$script_path" ]; then + # Pass the event_payload JSON via stdin so shell metacharacters + # ($VAR, backticks, ;, &&) don't fall through to argv (the + # #2741 / slice-5 motivating concern; even though this argv is + # composed entirely by the wrapper here, the stdin path keeps + # the surface honest and matches the slice-5 prose-arg rule). + # All four env vars (``EGG_AGENT_ROLE`` / ``EGG_BASE_BRANCH`` / + # ``EGG_REPO_PATH`` / ``EGG_BRC_MEMORY``) are read by the script + # from env directly. The prefix MUST attach to ``python3`` (RHS), + # not ``printf`` (LHS) -- the earlier form attached only to + # ``printf`` and ``python3`` inherited from the parent shell. + # ``EGG_REPO_PATH`` is re-exported explicitly here + # (reviewer_holistic v2 #2): the script falls back to + # ``os.getcwd()`` when unset, but propagating the + # orchestrator-set value keeps the wrapper symmetric and immune + # to an unset-in-parent edge case. Capture stderr to a temp + # file so the cw_log fallback surfaces the first line of the + # failure (script-not-found vs schema-drift vs crash otherwise + # indistinguishable). + local err_tmp + err_tmp=$(mktemp -t event-prompt-stderr.XXXXXX 2>/dev/null || echo "/tmp/event-prompt-stderr-$$.log") + prompt=$(printf '%s' "$event_payload" \ + | EGG_AGENT_ROLE="$role" \ + EGG_BASE_BRANCH="$base_branch" \ + EGG_REPO_PATH="${EGG_REPO_PATH:-$PWD}" \ + EGG_BRC_MEMORY="${EGG_BRC_MEMORY:-full}" \ + python3 "$script_path" "$action" 2>"$err_tmp") + prompt_rc=$? + fi + + if [ "$prompt_rc" -ne 0 ] || [ -z "$prompt" ]; then + # Fallback prompt -- keep the event-pump moving rather than + # failing the agent invocation when the composer is unavailable + # (script missing, schema drift, transient git log failure). + # The idle-budget safety net catches a wedged event-pump even + # under a degraded composer; failing here would defeat that. + local err_head="" + if [ -n "${err_tmp:-}" ] && [ -r "$err_tmp" ]; then + err_head=$(head -1 "$err_tmp" 2>/dev/null) + fi + if [ -n "$err_head" ]; then + cw_log "compose_event_prompt unavailable (rc=$prompt_rc, stderr: $err_head); using slice-2 stub prompt." + else + cw_log "compose_event_prompt unavailable (rc=$prompt_rc); using slice-2 stub prompt." + fi + prompt=$(printf 'BRC event-pump handler\nRole: %s\nSlice: %s\nAction: %s\nEvent payload (JSON): %s\n\nHandle this single event according to the role contract, update durable BRC memory, then exit naturally. The wrapper will invoke you again with the next event.\n' \ + "$role" "$slice" "$action" "$event_payload") + fi + # Best-effort cleanup of the stderr capture file. The trap on the + # outer wrapper handles SIGTERM cleanup; this cleanup keeps a busy + # event-pump from accumulating stale per-invocation temp files. + if [ -n "${err_tmp:-}" ] && [ -e "$err_tmp" ]; then + rm -f "$err_tmp" 2>/dev/null || true + fi + # #3077 slice-1 task-1-1: prepend any sync-to-proposal failure + # banners the prior ``sync_to_proposals`` accumulated so the agent + # learns that its worktree may not reflect the producer's commit + # BEFORE it reads any local diff. The common (no-failure) path + # leaves ``SYNC_FAILURE_BANNERS`` empty so the prompt handed to the + # agent is byte-identical to the slice-0 prompt. Re-clear after + # consumption so a subsequent ``propose`` arm (which intentionally + # skips the sync per R11a) can never inherit stale banners. + if [ -n "${SYNC_FAILURE_BANNERS:-}" ]; then + prompt="${SYNC_FAILURE_BANNERS}${prompt}" + SYNC_FAILURE_BANNERS="" + fi + python3 -m egg_agent --model opus --max-turns 1000 "$prompt" +} + +# Sync-to-proposal (#3076 / #3077 clause 2): before a review invocation +# (ack/nack), merge each pending producer's proposed commit into this +# reviewer's worktree so reviewers that must RUN the proposal (tester) +# have a real checkout. This is deterministic wrapper bash replacing +# the fetch/merge prose that previously lived in spawn prompts the +# event pump provably discards (#3033). Fail-soft at every step: an +# unresolvable SHA, a conflicting merge, or a dirty tree logs and +# continues — the per-event prompt's `git show :` reads +# (#3078) work from the shared object store either way, so the agent +# is never blocked on this sync succeeding. +# +# NB: ``git merge --no-edit`` produces a real merge commit (or fast- +# forwards) — HEAD advances and is NOT reset between events. Two +# durable side effects follow: +# - Dual-role agents (e.g. tester acting as producer-then-reviewer- +# then-producer): a subsequent producer turn commits on top of an +# ancestry that contains peer proposal commits. The producer arm +# (R11a, see the dispatcher below) intentionally skips the sync, +# but it does NOT un-merge syncs from prior reviewer arms. +# - Multiple producers per event: SHAs are merged sequentially. A +# conflict on the second SHA aborts that merge but leaves the +# first merge intact on HEAD. +# Neither is wrong — the per-event-prompt ``git show :`` +# fallback covers the failure path either way — but this is durable +# HEAD mutation, not transient enrichment. +# Per-event accumulator for sync-to-proposal failure banners +# (#3077 slice-1 task-1-1). Reset at the top of every +# ``sync_to_proposals`` call; appended to per failed SHA; consumed +# (and re-cleared) by ``invoke_agent_for_event`` which prepends the +# accumulated banner text to the composed event prompt BEFORE handing +# it to the agent. The default empty value keeps the +# ``merged``/``already-ancestor`` (no-failure) path byte-identical to +# the slice-0 prompt — the prepend is a no-op when this variable is +# empty. +SYNC_FAILURE_BANNERS="" + +sync_to_proposals() { + # #3216 (WS1 of #3209): only reviewers that EXECUTE the proposal (run + # the test suite / build against the merged tree) need a working-tree + # merge. Every other reviewer reads peer artifacts via this prompt's + # ``git show`` / ``egg-artifact`` served reads, so merging the peer's + # *whole* tree into their worktree buys nothing and risks the dual-role + # criss-cross propagation that corrupts shared drafts (#3208: a reviewer + # arm merges a peer's plan.md, a later producer turn commits on top, and + # the cross-merged lineages spawn spurious conflicts / a rebase-mangled + # yaml fence / a "File modified since read" livelock). Allowed roles are + # rendered from ``REVIEWER_CHECKOUT_ROLE_VALUES``; default-deny so an + # unset/unknown role reads via git-show (the safe, non-replicating side). + local _role="${EGG_AGENT_ROLE:-}" + case " tester " in + *" $_role "*) : ;; # role runs the proposal; a real merged checkout is required + *) + SYNC_FAILURE_BANNERS="" + cw_log "sync-to-proposal: role=$_role reads peer artifacts via git-show/egg-artifact; skipping working-tree merge (#3216)." + return 0 + ;; + esac + local event_payload="$1" + local repo="${EGG_REPO_PATH:-$PWD}" + local shas sha + # Reset the per-event failure accumulator at the top of each call + # so banners from a prior event never leak into the next prompt. + SYNC_FAILURE_BANNERS="" + # Extract pending_reviews[].proposal_commit_sha, strictly hex- + # validated (7-64 chars) before any git interpolation. The payload + # is orchestrator-composed, but the producer-supplied SHA rides + # through it, so revalidate at the consumer — same stance as + # event_prompt.py's _extract_proposal_sha_for_producer. The + # ProposalPayload writer-side check also admits non-hex sentinels + # like RECONSTRUCTED_NO_SHA; the hex requirement here filters those + # out (there is nothing to merge for a reconstructed proposal). + shas=$(printf '%s' "$event_payload" | python3 -c " +import sys, json, re +try: + d = json.load(sys.stdin) +except Exception: + sys.exit(0) +pat = re.compile(r'[0-9a-fA-F]{7,64}') +out = [] +for pr in (d.get('pending_reviews') or []): + if not isinstance(pr, dict): + continue + sha = str(pr.get('proposal_commit_sha') or '') + if pat.fullmatch(sha) and sha not in out: + out.append(sha) +print('\n'.join(out)) +" 2>/dev/null) + [ -z "$shas" ] && return 0 + while IFS= read -r sha; do + [ -z "$sha" ] && continue + if ! git -C "$repo" cat-file -e "$sha^{commit}" 2>/dev/null; then + # Per-role worktrees share the host repo's object store, so + # the SHA normally resolves without network; a best-effort + # fetch covers split-object-store runtimes. + git -C "$repo" fetch --quiet origin >/dev/null 2>&1 || true + fi + if ! git -C "$repo" cat-file -e "$sha^{commit}" 2>/dev/null; then + cw_log "sync-to-proposal: $sha unresolvable in $repo; reviewer falls back to the prompt's git-show reads." + cw_log "sync-to-proposal: outcome=unresolvable sha=$sha" + # ``unresolvable`` failure: append a NOT-synced banner so + # the agent treats its local diff as unreliable for this + # SHA and falls back to the rendered ``git show`` commands. + SYNC_FAILURE_BANNERS+="> **WARNING:** worktree NOT synced to \`$sha\` (\`unresolvable\`); treat your local diff as unreliable — use the rendered \`git log\` / \`git show\` fallback commands in this prompt instead."$'\n\n' + continue + fi + if git -C "$repo" merge-base --is-ancestor "$sha" HEAD 2>/dev/null; then + cw_log "sync-to-proposal: outcome=already-ancestor sha=$sha" + continue + fi + if git -C "$repo" merge --no-edit "$sha" >/dev/null 2>&1; then + cw_log "sync-to-proposal: merged proposal commit $sha into the worktree." + cw_log "sync-to-proposal: outcome=merged sha=$sha" + else + git -C "$repo" merge --abort >/dev/null 2>&1 || true + cw_log "sync-to-proposal: merge of $sha failed (conflict or dirty tree); aborted — reviewer reads via git show instead." + cw_log "sync-to-proposal: outcome=merge-failed sha=$sha" + # ``merge-failed`` failure: same banner shape as + # ``unresolvable`` so the agent learns NOT to trust the + # local diff and falls back to ``git show``. The merge + # was already aborted above so wrapper state is clean. + SYNC_FAILURE_BANNERS+="> **WARNING:** worktree NOT synced to \`$sha\` (\`merge-failed\`); treat your local diff as unreliable — use the rendered \`git log\` / \`git show\` fallback commands in this prompt instead."$'\n\n' + fi + done <<< "$shas" + return 0 +} + +# --- one-shot event handler (#3164) ------------------------------------ +ONE_SHOT_ACTION="${EGG_EVENT_ACTION:-}" +if [ -z "$ONE_SHOT_ACTION" ]; then + cw_log "FATAL: no EGG_EVENT_ACTION injected. Under the orchestrator-owned event loop (#3164) every agent pod is spawned one-shot per BRC event; a wrapper with no injected action is a caller bug. Refusing (exit 64)." + exit 64 +fi +cw_log "One-shot event handler engaged (action=$ONE_SHOT_ACTION, dedupe=${EGG_EVENT_DEDUPE_KEY:-none}, role=${EGG_AGENT_ROLE:-?}, slice=${EGG_SLICE_ID:-none})." + +case "$ONE_SHOT_ACTION" in + confirm|complete) + # confirm/complete are agent-free and run orchestrator-side (no + # pod is ever spawned for them). Reaching a wrapper with one + # injected is a caller bug -- reject loudly. + cw_log "FATAL: action=$ONE_SHOT_ACTION must run orchestrator-side with no pod and must never be injected into a wrapper. Refusing (exit 64)." + exit 64 + ;; + propose|ack|nack) + : ;; + *) + cw_log "FATAL: unknown injected EGG_EVENT_ACTION='$ONE_SHOT_ACTION' (expected propose|ack|nack). Refusing (exit 64)." + exit 64 + ;; +esac + +# Single foreground liveness ping (no background emitter): the +# silent-mid-event tripwire keys on heartbeats from an active one-shot pod. +emit_heartbeat "WORKING" "one-shot event handler action=$ONE_SHOT_ACTION" + +# Stale-event backstop: re-derive next-action ONCE. If consensus has moved +# on (derived action != injected event) this spawn is a duplicate/stale +# delivery -- exit 0 WITHOUT invoking the agent. The spawner dedupes on the +# derived event, but a race can still deliver a stale one; this is the +# in-pod backstop for that race. +ONE_SHOT_NEXT_JSON=$(fetch_next_action) +ONE_SHOT_FETCH_RC=$? +ONE_SHOT_DERIVED=$(next_action_field "$ONE_SHOT_NEXT_JSON" "action") +if [ "$ONE_SHOT_FETCH_RC" -ne 0 ]; then + # The re-check itself did NOT cleanly succeed. ``fetch_next_action`` + # returns its ``{"action":"wait"}`` fallback on ANY non-zero rc (409 + # stale_version / aggregated-NACK barrier, 5xx, transport failure + # alike), so we cannot distinguish a genuinely stale event from a + # transient blip. Exiting 0 here would report a "clean handoff" and let + # a transient blip silently DROP a live event. Instead exit 75 + # (EX_TEMPFAIL): a distinct code telling the orchestrator supervisor the + # freshness re-check was inconclusive, so it must re-derive next-action + # rather than treat the event as handled. + cw_log "One-shot re-check did not cleanly succeed (fetch rc=$ONE_SHOT_FETCH_RC); cannot confirm event freshness. Exiting 75 (EX_TEMPFAIL) so the supervisor re-derives rather than assuming a clean handoff." + exit 75 +fi +if [ "$ONE_SHOT_DERIVED" != "$ONE_SHOT_ACTION" ]; then + # Re-check succeeded and the live action has positively moved on: a + # genuinely stale/duplicate delivery. exit 0 means "confirmed stale, no + # agent needed" -- NOT "could not tell". + cw_log "Injected event is stale (injected=$ONE_SHOT_ACTION, derived=${ONE_SHOT_DERIVED:-}); exiting 0 without invoking the agent." + exit 0 +fi + +# Use the freshly-derived event_payload (current truth) for the invocation. +# Reviewer arms (ack/nack) sync the pending proposal commits into the +# worktree first (#3076/#3077); the producer ``propose`` arm intentionally +# does not (R11a -- a producer's own commits are already on HEAD). +ONE_SHOT_PAYLOAD=$(next_action_field "$ONE_SHOT_NEXT_JSON" "event_payload") +if [ "$ONE_SHOT_ACTION" = "ack" ] || [ "$ONE_SHOT_ACTION" = "nack" ]; then + sync_to_proposals "$ONE_SHOT_PAYLOAD" +fi + +one_shot_start=$SECONDS +invoke_agent_for_event "$ONE_SHOT_ACTION" "$ONE_SHOT_PAYLOAD" +one_shot_rc=$? +one_shot_secs=$(( SECONDS - one_shot_start )) +cw_log "One-shot invocation done (action=$ONE_SHOT_ACTION, rc=$one_shot_rc, duration=${one_shot_secs}s); exiting with the agent rc." +# Exit-code contract for the orchestrator supervisor: 75/EX_TEMPFAIL is +# RESERVED for the inconclusive-re-check outcome above ("freshness could +# not be confirmed, re-derive next-action"). Agent passthrough rcs do not +# include 75, so the two meanings never collide. +exit "$one_shot_rc" diff --git a/orchestrator/tests/golden/event_pump_wrapper_pod_default.sh.golden b/orchestrator/tests/golden/event_pump_wrapper_pod_default.sh.golden deleted file mode 100644 index 0e905f2b06..0000000000 --- a/orchestrator/tests/golden/event_pump_wrapper_pod_default.sh.golden +++ /dev/null @@ -1,825 +0,0 @@ -#!/bin/bash -set -uo pipefail - -# Event-pump wrapper (#2908 slice-2). Deterministic loop driven by -# ``egg-orch brc get-state`` + ``egg-orch brc next-action``; the agent -# is invoked one-shot per actionable event rather than holding a -# blocking wait. Migrated wrapper-owned heartbeats (#2036, #2451) -# replace the agent-side liveness path. - -IDLE_BUDGET_MIN="${EGG_BRC_IDLE_BUDGET_MIN:-30}" -IDLE_BUDGET_SECS=$(( IDLE_BUDGET_MIN * 60 )) -HB_INTERVAL_SECS="${EGG_BRC_HEARTBEAT_INTERVAL_SECS:-30}" -WAIT_TIMEOUT_SECS="${EGG_BRC_WAIT_TIMEOUT_SECS:-60}" - -# Wrapper-owned background heartbeat PID. ``cleanup`` (installed below) -# kills it on EXIT so SIGTERM from the orchestrator does not leave a -# stray background process holding the gateway session open. -HB_BG_PID="" - -cw_log() { - echo "[event-pump] $*" >&2 -} - -# --- Agent-invocation exit-code classifiers (#2908 task-4-2) ----------- -# -# Migrated from the legacy capped-restart template. The event-pump -# invokes the agent one-shot per actionable event (``propose|ack|nack`` -# arm in the loop below); a non-zero exit there is still meaningful in -# the same three ways the legacy template distinguished: -# -# * ``is_buffer_overflow`` — the Claude Agent SDK 1 MiB JSON -# message-reader overflow signature (issue #2804). Deterministic -# under one-shot too: the next invocation hits the same oversized -# tool result. The event-pump's idle-budget safety net catches this -# eventually, but the classifier lets the operator alert sooner with -# a more specific anomaly tag. -# * ``is_transient_crash`` — signal-based exits (SIGABRT, SIGFPE, -# SIGKILL/OOM, SIGSEGV, Bun segfault) where a retry is appropriate. -# * ``is_startup_failure`` — exit 1 within the -# ``$STARTUP_FAILURE_WINDOW_SECONDS`` window (SDK API/network blip -# manifesting as exit 1 before the agent did meaningful work). -# -# The event-pump's ``propose|ack|nack`` arm does NOT yet branch on -# these signals (the consecutive-failure counter + idle-budget alert -# combo handles the operator-visible escalation today). Keeping them as -# named helpers preserves a clean hook for future revisions (per task-4-2 -# acceptance: classifiers "are still valid signals under the new idle/ -# no-progress safety budget"). -STARTUP_FAILURE_WINDOW_SECONDS=30 - -is_buffer_overflow() { - [ -f "${AGENT_OUTPUT_LOG:-}" ] || return 1 - grep -q "exceeded maximum buffer size" "$AGENT_OUTPUT_LOG" 2>/dev/null -} - -is_transient_crash() { - local code="$1" - case "$code" in - 134|136|137|139|255) return 0 ;; # SIGABRT, SIGFPE, SIGKILL/OOM, SIGSEGV, Bun segfault - *) return 1 ;; - esac -} - -is_startup_failure() { - local code="$1" - local duration="$2" - if [ "$code" -ne 1 ]; then - return 1 - fi - if [ "$duration" -lt "$STARTUP_FAILURE_WINDOW_SECONDS" ]; then - return 0 - fi - return 1 -} - -# ----------------------------------------------------------------------- - -# Emit one heartbeat. The CLI's ``message heartbeat`` handler auto- -# attaches ``slice_id`` from ``$EGG_SLICE_ID`` via -# ``_maybe_attach_slice_id`` in -# ``sandbox/egg_agent_tools/handlers/_gateway.py`` -- this is the -# #2451 migration: every wrapper heartbeat refreshes the slice-scoped -# gateway session as a side effect. We also echo the slice tag in the -# ``--body`` so a snapshot test (#2908 task-2-6 (ii)) can grep for the -# slice_id propagation without intercepting the HTTP POST. -emit_heartbeat() { - local state="$1" - local body_text="$2" - local slice_tag="${EGG_SLICE_ID:-none}" - timeout 5 egg-orch message heartbeat \ - --state "$state" \ - --body "$body_text (slice=$slice_tag)" \ - >/dev/null 2>&1 || true -} - -# Start a background heartbeat emitter for the duration of a blocking -# call. Replaces ``handlers/message.py:_start_wait_loop_heartbeat`` -- -# the wrapper now owns this responsibility (#2908 task-2-2). -start_background_heartbeat() { - local body_text="$1" - ( - # Install a TERM trap that exits the subshell cleanly so the - # outer ``stop_background_heartbeat``'s ``kill $HB_BG_PID`` - # (default signal SIGTERM) reaps the child rather than - # deadlocking on ``wait``. The earlier ``trap '' TERM`` form - # MASKED the signal and caused a wait deadlock that hung the - # whole event-pump after the first wait-loop return - # (reviewer_concurrency v1 finding 1 / #2908 slice-2 NACK). - # ``set -uo pipefail`` (without ``-e``) does not propagate - # subshell failures to the parent; ``emit_heartbeat``'s - # ``|| true`` already swallows any CLI error -- so there is no - # signal-defense to install here. - trap 'exit 0' TERM - while true; do - sleep "$HB_INTERVAL_SECS" - emit_heartbeat "WAITING_FOR_EVENT" "$body_text" - done - ) & - HB_BG_PID=$! -} - -stop_background_heartbeat() { - if [ -n "$HB_BG_PID" ]; then - # Use SIGTERM (default ``kill`` signal) so the subshell's - # ``trap 'exit 0' TERM`` exits the heartbeat loop cleanly, - # then ``wait`` reaps it without blocking. SIGKILL would - # work too but loses the chance to log a final exit code. - kill "$HB_BG_PID" 2>/dev/null || true - wait "$HB_BG_PID" 2>/dev/null || true - HB_BG_PID="" - fi -} - -cleanup() { - stop_background_heartbeat -} -trap cleanup EXIT TERM INT - -# Fetch the BRC consensus state. Returns ``{}`` on any failure so -# downstream parsers can short-circuit without a Python crash. -fetch_state() { - egg-orch brc get-state --json 2>/dev/null || echo "{}" -} - -# Has the role for this pod already reached CONFIRMED in the BRC -# matrix? Used to decide whether to include CONSENSUS_CONFIRMED in -# the wait-filter set (risk_analyst R12, orchestrator HTTP-400 rule -# from #2064 / #2482). -role_is_confirmed() { - local state_json="$1" - echo "$state_json" | python3 -c " -import sys, json, os -role = os.environ.get('EGG_AGENT_ROLE', '') -try: - d = json.load(sys.stdin) -except Exception: - print('False'); sys.exit(0) -agents = (d.get('consensus') or {}).get('agents') or {} -print('True' if agents.get(role, {}).get('confirmed') else 'False') -" 2>/dev/null || echo "False" -} - -# Has global consensus already completed? Lets the loop short-circuit -# (e.g. when another role's confirmation flipped is_complete after we -# CONFIRMED but before SIGTERM landed). -consensus_is_complete() { - local state_json="$1" - echo "$state_json" | python3 -c " -import sys, json -try: - d = json.load(sys.stdin) -except Exception: - print('False'); sys.exit(0) -print('True' if (d.get('consensus') or {}).get('is_complete') else 'False') -" 2>/dev/null || echo "False" -} - -# Ask the orchestrator route what to do next. Returns a JSON document -# with ``action`` ('wait'|'propose'|'ack'|'nack'|'confirm'|'complete') -# and an optional ``event_payload``. -# -# Per #2908 task-2-1 (acceptance: "Wrapper handles 409 stale_version -# and 409 aggregated-NACK from ``brc next-action`` as event-pump -# signals (re-fetch state, re-invoke), NOT as transient crashes to -# retry with backoff."), HTTP 409 from the route is a signal to -# re-fetch state, not a crash. The CLI surfaces 409 as a non-zero -# exit code with an empty/invalid JSON; falling back to ``{"action":"wait"}`` -# lets the next loop iteration call ``brc get-state`` again, which -# observes the new state and emits the correct next action. -fetch_next_action() { - local out rc - out=$(egg-orch brc next-action --role "${EGG_AGENT_ROLE:-unknown}" --json 2>/dev/null) - rc=$? - if [ "$rc" -eq 0 ] && [ -n "$out" ]; then - echo "$out" - return 0 - fi - # Fallback so the main loop keeps blocking on the bus and re-derives - # state on the next iteration. The CLI returns non-zero for any of: - # 409 stale_version, 409 aggregated-NACK barrier (both expected - # event-pump signals -- the next loop's ``brc get-state`` observes - # the changed state), 5xx, transport failure. Log it so operators - # reading wrapper logs can distinguish "orchestrator returned 409" - # (benign, expected) from "transport unreachable" (worth checking) - # without correlating against the orchestrator's audit log - # (tester v1 non-blocker #3). We always echo the fallback JSON so - # the next-action parser doesn't crash; we propagate the original - # ``rc`` as the function's exit code (caller reads ``$?`` after the - # ``$(...)`` substitution to count the streak per reviewer §3). - cw_log "brc next-action returned rc=$rc / empty body; falling back to {\"action\":\"wait\"} and re-deriving state next loop." - echo '{"action":"wait"}' - return "$rc" -} - -# Extract a top-level scalar field from a next-action JSON document. -# Nested objects (event_payload) are re-serialised as JSON so the -# bash caller can pass them downstream verbatim. -next_action_field() { - local action_json="$1" - local field="$2" - echo "$action_json" | python3 -c " -import sys, json -field = sys.argv[1] -try: - d = json.load(sys.stdin) -except Exception: - print(''); sys.exit(0) -v = d.get(field) -if isinstance(v, (dict, list)): - print(json.dumps(v)) -elif v is None: - print('') -else: - print(v) -" "$field" 2>/dev/null || echo "" -} - -# Build the typed wait-filter set. Pre-confirm waits MUST omit -# CONSENSUS_CONFIRMED -- the orchestrator rejects that filter -# combination with HTTP 400 (#2064, #2482, risk_analyst R12). The -# six-event set is the union the architect's -# ``verification_strategy.slice_2.i`` snapshot test pins. -build_wait_args() { - local include_confirmed="$1" - local args=( - --for CONSENSUS_PROPOSE - --for CONSENSUS_ACK - --for CONSENSUS_NACK - --for STATUS - --for CONSENSUS_RE_REVIEW - --for OVERSEER_ALERT - ) - if [ "$include_confirmed" = "True" ]; then - args+=( --for CONSENSUS_CONFIRMED ) - fi - args+=( --timeout "$WAIT_TIMEOUT_SECS" ) - printf '%s\n' "${args[@]}" -} - -# Block on the orchestrator message bus for the next BRC event while -# the wrapper-owned background heartbeat emitter keeps the overseer's -# liveness tracker and the gateway-session idle timer happy. -wait_for_event() { - local include_confirmed="$1" - local hb_body="event-pump wait role=${EGG_AGENT_ROLE:-?}" - mapfile -t WAIT_ARGS < <(build_wait_args "$include_confirmed") - start_background_heartbeat "$hb_body" - emit_heartbeat "WAITING_FOR_EVENT" "$hb_body" - egg-orch message wait-loop "${WAIT_ARGS[@]}" --max-iterations 1 >/dev/null 2>&1 - local rc=$? - stop_background_heartbeat - emit_heartbeat "WORKING" "event-pump woke (rc=$rc)" - return $rc -} - -# Invoke the agent one-shot with the per-event prompt. Slice-3 -# (TASK-3-1 / TASK-3-2) replaces the slice-2 stub with the full -# ``compose_event_prompt`` payload: memory excerpt (when -# ``EGG_BRC_MEMORY=full``) + per-producer -# ``git log {sha}..HEAD --not origin/{base} -p`` delta + NACK -# payload + the contract's ``task_description`` (#3123, read from the -# worktree contract file via the pod-inherited ``EGG_PIPELINE_ID`` / -# ``EGG_ISSUE_NUMBER``). The composer lives at -# ``/opt/egg-runtime/orchestrator/routes/event_prompt.py`` -- the -# wrapper invokes its ``if __name__ == '__main__'`` CLI directly so the -# heavy ``orchestrator.routes`` package ``__init__.py`` (Flask import) -# is bypassed. ``EGG_EVENT_PROMPT_SCRIPT`` overrides the path for tests. -# -# When the composer fails (script missing, malformed memory file, git -# log subprocess crash) we fall back to the slice-2 minimal stub so the -# event-pump keeps running rather than failing the agent invocation. -# This is symmetric with the rest of the wrapper's "block, alert, -# continue" stance under the idle-budget safety net. -invoke_agent_for_event() { - local action="$1" - local event_payload="$2" - local role="${EGG_AGENT_ROLE:-unknown}" - local slice="${EGG_SLICE_ID:-none}" - local base_branch="${EGG_BASE_BRANCH:-main}" - local script_path="${EGG_EVENT_PROMPT_SCRIPT:-/opt/egg-runtime/orchestrator/routes/event_prompt.py}" - local prompt prompt_rc=1 - - if [ -r "$script_path" ]; then - # Pass the event_payload JSON via stdin so shell metacharacters - # ($VAR, backticks, ;, &&) don't fall through to argv (the - # #2741 / slice-5 motivating concern; even though this argv is - # composed entirely by the wrapper here, the stdin path keeps - # the surface honest and matches the slice-5 prose-arg rule). - # All four env vars (``EGG_AGENT_ROLE`` / ``EGG_BASE_BRANCH`` / - # ``EGG_REPO_PATH`` / ``EGG_BRC_MEMORY``) are read by the script - # from env directly. The prefix MUST attach to ``python3`` (RHS), - # not ``printf`` (LHS) -- the earlier form attached only to - # ``printf`` and ``python3`` inherited from the parent shell. - # ``EGG_REPO_PATH`` is re-exported explicitly here - # (reviewer_holistic v2 #2): the script falls back to - # ``os.getcwd()`` when unset, but propagating the - # orchestrator-set value keeps the wrapper symmetric and immune - # to an unset-in-parent edge case. Capture stderr to a temp - # file so the cw_log fallback surfaces the first line of the - # failure (script-not-found vs schema-drift vs crash otherwise - # indistinguishable). - local err_tmp - err_tmp=$(mktemp -t event-prompt-stderr.XXXXXX 2>/dev/null || echo "/tmp/event-prompt-stderr-$$.log") - prompt=$(printf '%s' "$event_payload" \ - | EGG_AGENT_ROLE="$role" \ - EGG_BASE_BRANCH="$base_branch" \ - EGG_REPO_PATH="${EGG_REPO_PATH:-$PWD}" \ - EGG_BRC_MEMORY="${EGG_BRC_MEMORY:-full}" \ - python3 "$script_path" "$action" 2>"$err_tmp") - prompt_rc=$? - fi - - if [ "$prompt_rc" -ne 0 ] || [ -z "$prompt" ]; then - # Fallback prompt -- keep the event-pump moving rather than - # failing the agent invocation when the composer is unavailable - # (script missing, schema drift, transient git log failure). - # The idle-budget safety net catches a wedged event-pump even - # under a degraded composer; failing here would defeat that. - local err_head="" - if [ -n "${err_tmp:-}" ] && [ -r "$err_tmp" ]; then - err_head=$(head -1 "$err_tmp" 2>/dev/null) - fi - if [ -n "$err_head" ]; then - cw_log "compose_event_prompt unavailable (rc=$prompt_rc, stderr: $err_head); using slice-2 stub prompt." - else - cw_log "compose_event_prompt unavailable (rc=$prompt_rc); using slice-2 stub prompt." - fi - prompt=$(printf 'BRC event-pump handler\nRole: %s\nSlice: %s\nAction: %s\nEvent payload (JSON): %s\n\nHandle this single event according to the role contract, update durable BRC memory, then exit naturally. The wrapper will invoke you again with the next event.\n' \ - "$role" "$slice" "$action" "$event_payload") - fi - # Best-effort cleanup of the stderr capture file. The trap on the - # outer wrapper handles SIGTERM cleanup; this cleanup keeps a busy - # event-pump from accumulating stale per-invocation temp files. - if [ -n "${err_tmp:-}" ] && [ -e "$err_tmp" ]; then - rm -f "$err_tmp" 2>/dev/null || true - fi - # #3077 slice-1 task-1-1: prepend any sync-to-proposal failure - # banners the prior ``sync_to_proposals`` accumulated so the agent - # learns that its worktree may not reflect the producer's commit - # BEFORE it reads any local diff. The common (no-failure) path - # leaves ``SYNC_FAILURE_BANNERS`` empty so the prompt handed to the - # agent is byte-identical to the slice-0 prompt. Re-clear after - # consumption so a subsequent ``propose`` arm (which intentionally - # skips the sync per R11a) can never inherit stale banners. - if [ -n "${SYNC_FAILURE_BANNERS:-}" ]; then - prompt="${SYNC_FAILURE_BANNERS}${prompt}" - SYNC_FAILURE_BANNERS="" - fi - python3 -m egg_agent --model opus --max-turns 1000 "$prompt" -} - -# Sync-to-proposal (#3076 / #3077 clause 2): before a review invocation -# (ack/nack), merge each pending producer's proposed commit into this -# reviewer's worktree so reviewers that must RUN the proposal (tester) -# have a real checkout. This is deterministic wrapper bash replacing -# the fetch/merge prose that previously lived in spawn prompts the -# event pump provably discards (#3033). Fail-soft at every step: an -# unresolvable SHA, a conflicting merge, or a dirty tree logs and -# continues — the per-event prompt's `git show :` reads -# (#3078) work from the shared object store either way, so the agent -# is never blocked on this sync succeeding. -# -# NB: ``git merge --no-edit`` produces a real merge commit (or fast- -# forwards) — HEAD advances and is NOT reset between events. Two -# durable side effects follow: -# - Dual-role agents (e.g. tester acting as producer-then-reviewer- -# then-producer): a subsequent producer turn commits on top of an -# ancestry that contains peer proposal commits. The producer arm -# (R11a, see the dispatcher below) intentionally skips the sync, -# but it does NOT un-merge syncs from prior reviewer arms. -# - Multiple producers per event: SHAs are merged sequentially. A -# conflict on the second SHA aborts that merge but leaves the -# first merge intact on HEAD. -# Neither is wrong — the per-event-prompt ``git show :`` -# fallback covers the failure path either way — but this is durable -# HEAD mutation, not transient enrichment. -# Per-event accumulator for sync-to-proposal failure banners -# (#3077 slice-1 task-1-1). Reset at the top of every -# ``sync_to_proposals`` call; appended to per failed SHA; consumed -# (and re-cleared) by ``invoke_agent_for_event`` which prepends the -# accumulated banner text to the composed event prompt BEFORE handing -# it to the agent. The default empty value keeps the -# ``merged``/``already-ancestor`` (no-failure) path byte-identical to -# the slice-0 prompt — the prepend is a no-op when this variable is -# empty. -SYNC_FAILURE_BANNERS="" - -sync_to_proposals() { - # #3216 (WS1 of #3209): only reviewers that EXECUTE the proposal (run - # the test suite / build against the merged tree) need a working-tree - # merge. Every other reviewer reads peer artifacts via this prompt's - # ``git show`` / ``egg-artifact`` served reads, so merging the peer's - # *whole* tree into their worktree buys nothing and risks the dual-role - # criss-cross propagation that corrupts shared drafts (#3208: a reviewer - # arm merges a peer's plan.md, a later producer turn commits on top, and - # the cross-merged lineages spawn spurious conflicts / a rebase-mangled - # yaml fence / a "File modified since read" livelock). Allowed roles are - # rendered from ``REVIEWER_CHECKOUT_ROLE_VALUES``; default-deny so an - # unset/unknown role reads via git-show (the safe, non-replicating side). - local _role="${EGG_AGENT_ROLE:-}" - case " tester " in - *" $_role "*) : ;; # role runs the proposal; a real merged checkout is required - *) - SYNC_FAILURE_BANNERS="" - cw_log "sync-to-proposal: role=$_role reads peer artifacts via git-show/egg-artifact; skipping working-tree merge (#3216)." - return 0 - ;; - esac - local event_payload="$1" - local repo="${EGG_REPO_PATH:-$PWD}" - local shas sha - # Reset the per-event failure accumulator at the top of each call - # so banners from a prior event never leak into the next prompt. - SYNC_FAILURE_BANNERS="" - # Extract pending_reviews[].proposal_commit_sha, strictly hex- - # validated (7-64 chars) before any git interpolation. The payload - # is orchestrator-composed, but the producer-supplied SHA rides - # through it, so revalidate at the consumer — same stance as - # event_prompt.py's _extract_proposal_sha_for_producer. The - # ProposalPayload writer-side check also admits non-hex sentinels - # like RECONSTRUCTED_NO_SHA; the hex requirement here filters those - # out (there is nothing to merge for a reconstructed proposal). - shas=$(printf '%s' "$event_payload" | python3 -c " -import sys, json, re -try: - d = json.load(sys.stdin) -except Exception: - sys.exit(0) -pat = re.compile(r'[0-9a-fA-F]{7,64}') -out = [] -for pr in (d.get('pending_reviews') or []): - if not isinstance(pr, dict): - continue - sha = str(pr.get('proposal_commit_sha') or '') - if pat.fullmatch(sha) and sha not in out: - out.append(sha) -print('\n'.join(out)) -" 2>/dev/null) - [ -z "$shas" ] && return 0 - while IFS= read -r sha; do - [ -z "$sha" ] && continue - if ! git -C "$repo" cat-file -e "$sha^{commit}" 2>/dev/null; then - # Per-role worktrees share the host repo's object store, so - # the SHA normally resolves without network; a best-effort - # fetch covers split-object-store runtimes. - git -C "$repo" fetch --quiet origin >/dev/null 2>&1 || true - fi - if ! git -C "$repo" cat-file -e "$sha^{commit}" 2>/dev/null; then - cw_log "sync-to-proposal: $sha unresolvable in $repo; reviewer falls back to the prompt's git-show reads." - cw_log "sync-to-proposal: outcome=unresolvable sha=$sha" - # ``unresolvable`` failure: append a NOT-synced banner so - # the agent treats its local diff as unreliable for this - # SHA and falls back to the rendered ``git show`` commands. - SYNC_FAILURE_BANNERS+="> **WARNING:** worktree NOT synced to \`$sha\` (\`unresolvable\`); treat your local diff as unreliable — use the rendered \`git log\` / \`git show\` fallback commands in this prompt instead."$'\n\n' - continue - fi - if git -C "$repo" merge-base --is-ancestor "$sha" HEAD 2>/dev/null; then - cw_log "sync-to-proposal: outcome=already-ancestor sha=$sha" - continue - fi - if git -C "$repo" merge --no-edit "$sha" >/dev/null 2>&1; then - cw_log "sync-to-proposal: merged proposal commit $sha into the worktree." - cw_log "sync-to-proposal: outcome=merged sha=$sha" - else - git -C "$repo" merge --abort >/dev/null 2>&1 || true - cw_log "sync-to-proposal: merge of $sha failed (conflict or dirty tree); aborted — reviewer reads via git show instead." - cw_log "sync-to-proposal: outcome=merge-failed sha=$sha" - # ``merge-failed`` failure: same banner shape as - # ``unresolvable`` so the agent learns NOT to trust the - # local diff and falls back to ``git show``. The merge - # was already aborted above so wrapper state is clean. - SYNC_FAILURE_BANNERS+="> **WARNING:** worktree NOT synced to \`$sha\` (\`merge-failed\`); treat your local diff as unreliable — use the rendered \`git log\` / \`git show\` fallback commands in this prompt instead."$'\n\n' - fi - done <<< "$shas" - return 0 -} - -# Idle / no-progress safety budget (#2908 task-2-3). Replaces the -# legacy capped-restart cap (deleted by task-4-2): if no actionable -# event arrives for the configured idle budget we raise an -# OVERSEER_ALERT but the loop keeps blocking (the legacy template -# would exit 1 -> FAILED at this point). -LAST_PROGRESS=$SECONDS -ALERTED_AT_BUDGET=false -ALERTED_AT_DOUBLE=false - -# Consecutive-failure counters for the action arms (reviewer §1). -# Used to apply linear backoff on the ``confirm`` arm and to surface -# a distinguishable log when an action is persistently failing. -# The counters are arm-cluster scoped: they reset to 0 at the top of the -# loop when the next-action transitions away from the arm they apply to -# (reviewer §2 follow-up). That way a long-ago confirm-failure streak -# doesn't pre-load the backoff for a fresh confirm attempt hours later -# after the role briefly transitioned through ``wait`` / ``propose``. -CONFIRM_FAIL_STREAK=0 -AGENT_FAIL_STREAK=0 -# Sticky latches for the agent-invocation fail streak (#3138). Like the -# ``NEXT_ACTION_ALERTED_*`` latches below they are wrapper-lifetime -# sticky (the streak counter itself resets via the arm-cluster dispatch -# at the top of the loop, but a re-fired warning after a brief recovery -# is noise, not signal). The 10-streak latch escalates to a real -# overseer alert -- a streak of fast-failing invocations is a strong -# permanent/configuration-class signal (unknown model alias, auth -# misconfiguration, prompt-rendering crash) that the operator should -# hear about from the wrapper itself, not minutes later via the -# overseer's generic heartbeat-silence anomaly. -AGENT_FAIL_ALERTED_5=false -AGENT_FAIL_ALERTED_10=false -# Consecutive failures from ``fetch_next_action`` (reviewer §3): used -# to surface a distinguishable "many consecutive 5xx/transport failures" -# log line so an unhealthy orchestrator is differentiable from a benign -# 409 stale_version that re-derives on the next loop. Latches are -# sticky for the wrapper lifetime so the log doesn't re-fire if the -# counter happens to land back on the threshold after a brief recovery. -NEXT_ACTION_FAIL_STREAK=0 -NEXT_ACTION_ALERTED_5=false -NEXT_ACTION_ALERTED_20=false - -raise_idle_alert() { - local idle="$1" - local priority="$2" - local summary_extra="$3" - # Snapshot the current BRC state for the alert detail so operators - # see the consensus.agents. matrix without having to query - # pipeline status separately. Plan TASK-2-3 acceptance line: - # "alert payload includes anomaly type, priority, current BRC - # state" (tester v1 non-blocker #2). - local brc_snapshot snapshot_input - # The naive ``echo $VAR_OR_EMPTY_JSON | python3 ...`` form using a - # parameter-expansion default containing literal braces is unsafe: - # bash parses the brace inside the default greedily and leaves a - # trailing literal close-brace appended to the expanded value, - # corrupting JSON when STATE_JSON is set (tester v2 NACK finding). - # Use a separate variable and an explicit empty-string check so - # bash never sees unbalanced braces inside a parameter expansion. - snapshot_input="${STATE_JSON-}" - if [ -z "$snapshot_input" ]; then - snapshot_input='{}' - fi - brc_snapshot=$(printf '%s' "$snapshot_input" | python3 -c " -import sys, json, os -role = os.environ.get('EGG_AGENT_ROLE', '') -try: - d = json.load(sys.stdin) -except Exception: - print('(unavailable)'); sys.exit(0) -agents = (d.get('consensus') or {}).get('agents') or {} -my = agents.get(role) or {} -blocking = (d.get('consensus') or {}).get('blocking_agents') or [] -print(f\"role={role} producer_phase={my.get('producer_phase','?')} reviewer_phase={my.get('reviewer_phase','?')} confirmed={my.get('confirmed','?')} blocking_agents={blocking}\") -" 2>/dev/null || echo "(snapshot unavailable)") - timeout 5 egg-orch overseer alert "${EGG_PIPELINE_ID:-unknown}" \ - --role "${EGG_AGENT_ROLE:-agent}" \ - --anomaly stuck-phase-transition \ - --priority "$priority" \ - --summary "BRC event-pump idle for ${idle}s$summary_extra" \ - --detail "Event-pump for role=${EGG_AGENT_ROLE:-agent} slice=${EGG_SLICE_ID:-none} has seen no actionable BRC event for ${idle}s (configured budget ${IDLE_BUDGET_SECS}s). The loop continues blocking; no FAILED transition is forced. BRC state: $brc_snapshot" \ - >/dev/null 2>&1 || true -} - -raise_agent_fail_alert() { - local streak="$1" - local last_rc="$2" - local last_secs="$3" - local action="$4" - # Duration-aware classification (#3138): a fast failure means the - # invocation died before/at SDK init -- unknown model alias, auth - # misconfiguration, prompt-rendering crash -- i.e. a permanent - # configuration-class failure, not a transient. Name that in the - # alert so the operator doesn't have to infer it from cadence. The - # wrapper still never self-FAILs (post-#2908 design): the kill - # decision stays with the operator. - # - # The ``-le 2`` threshold (not ``< 1``) is deliberate: ``$SECONDS`` - # has whole-second granularity and ``invoke_agent_for_event`` includes - # prompt-composer overhead, so a genuine pre-SDK-init crash routinely - # measures 1-2s. Do not tighten this to ``< 1`` -- it would misclassify - # those crashes as transient. - local classification="repeated agent-invocation failure; attempts are taking ${last_secs}s so this may be transient (API/quota/transport)" - if [ "$last_secs" -le 2 ]; then - classification="attempts are fast-failing (${last_secs}s, before/at SDK init) -- likely a permanent configuration-class failure (unknown model alias, auth misconfiguration, prompt-rendering crash)" - fi - timeout 5 egg-orch overseer alert "${EGG_PIPELINE_ID:-unknown}" \ - --role "${EGG_AGENT_ROLE:-agent}" \ - --anomaly agent-invocation-fail-streak \ - --priority high \ - --summary "agent invocation failing repeatedly (action=${action}, streak=${streak})" \ - --detail "Event-pump for role=${EGG_AGENT_ROLE:-agent} slice=${EGG_SLICE_ID:-none} has had ${streak} consecutive agent-invocation failures on action=${action} (last rc=${last_rc}): ${classification}. The loop keeps retrying with linear backoff (capped 30s); no FAILED transition is forced. Idle budget continues to accrue." \ - >/dev/null 2>&1 || true -} - -check_idle_budget() { - local idle=$(( SECONDS - LAST_PROGRESS )) - local double=$(( 2 * IDLE_BUDGET_SECS )) - if [ "$idle" -ge "$double" ] && [ "$ALERTED_AT_DOUBLE" != "true" ]; then - cw_log "Idle 2x budget exceeded (${idle}s >= ${double}s); raising HIGH overseer alert." - raise_idle_alert "$idle" "high" " (2x budget)" - ALERTED_AT_DOUBLE=true - # When the loop jumps straight from idle=0 to >=2x budget (e.g. - # the wrapper paused for 60+ min between checks), set the 1x - # latch too -- the 2x alert subsumes the 1x notification, so - # the next ``check_idle_budget`` should not re-fire the 1x - # branch (tester v1 non-blocker #1). - ALERTED_AT_BUDGET=true - elif [ "$idle" -ge "$IDLE_BUDGET_SECS" ] && [ "$ALERTED_AT_BUDGET" != "true" ]; then - cw_log "Idle budget exceeded (${idle}s >= ${IDLE_BUDGET_SECS}s); raising overseer alert." - raise_idle_alert "$idle" "high" "" - ALERTED_AT_BUDGET=true - fi -} - -note_progress() { - LAST_PROGRESS=$SECONDS - ALERTED_AT_BUDGET=false - # Reviewer §6 nit: ``ALERTED_AT_DOUBLE`` is sticky for the lifetime - # of the loop. Once the operator has been paged at 2x budget, - # re-arming a fresh 1x alert later (after a single spurious - # ``note_progress``) is noise, not signal. -} - -# --- main event-pump loop --- -cw_log "Event-pump starting (role=${EGG_AGENT_ROLE:-?}, slice=${EGG_SLICE_ID:-none}, idle-budget=${IDLE_BUDGET_MIN}m)" -emit_heartbeat "WORKING" "event-pump start" - -while true; do - STATE_JSON=$(fetch_state) - - if [ "$(consensus_is_complete "$STATE_JSON")" = "True" ]; then - cw_log "Global consensus complete; exiting cleanly." - exit 0 - fi - - ROLE_CONFIRMED=$(role_is_confirmed "$STATE_JSON") - - ACTION_JSON=$(fetch_next_action) - NEXT_ACTION_RC=$? - if [ "$NEXT_ACTION_RC" -eq 0 ]; then - NEXT_ACTION_FAIL_STREAK=0 - else - # Reviewer §3 (minor): the CLI surfaces 409 stale_version, 409 - # aggregated-NACK barrier, 5xx, and transport failure all as the - # same non-zero rc + empty body. A single non-zero rc is benign - # (expected event-pump signal). A *streak* is an orchestrator- - # health signal worth surfacing separately so operators reading - # wrapper logs can tell a 409-stuck role from an unhealthy - # orchestrator without correlating against the audit log. - # - # Use ``-ge`` with sticky latches (rather than ``-eq``) so the log - # is robust to any future change in how the counter advances and - # the warning fires the first time the threshold is crossed - # without re-firing on every iteration past it. - NEXT_ACTION_FAIL_STREAK=$(( NEXT_ACTION_FAIL_STREAK + 1 )) - if [ "$NEXT_ACTION_FAIL_STREAK" -ge 5 ] && [ "$NEXT_ACTION_ALERTED_5" != "true" ]; then - cw_log "brc next-action has returned non-zero ${NEXT_ACTION_FAIL_STREAK} times in a row -- orchestrator may be unhealthy (5xx loop / transport down), not just a benign 409 stale_version. Idle budget continues to accrue." - NEXT_ACTION_ALERTED_5=true - fi - if [ "$NEXT_ACTION_FAIL_STREAK" -ge 20 ] && [ "$NEXT_ACTION_ALERTED_20" != "true" ]; then - cw_log "brc next-action has returned non-zero ${NEXT_ACTION_FAIL_STREAK} times in a row -- orchestrator may be unhealthy (5xx loop / transport down), not just a benign 409 stale_version. Idle budget continues to accrue." - NEXT_ACTION_ALERTED_20=true - fi - fi - ACTION=$(next_action_field "$ACTION_JSON" "action") - EVENT_PAYLOAD=$(next_action_field "$ACTION_JSON" "event_payload") - - # Reviewer §2 (non-blocking): the per-arm failure counters are - # arm-cluster scoped, not wrapper-lifetime. When the orchestrator - # transitions the role to a different action verb (e.g., a stuck - # ``confirm`` recovers via a fresh ``propose`` after a re-review - # event), reset the streak for the arm we just left so a brand-new - # attempt isn't pre-loaded with an old streak's backoff cap. - if [ "$ACTION" != "confirm" ]; then - CONFIRM_FAIL_STREAK=0 - fi - case "$ACTION" in - propose|ack|nack) ;; - *) AGENT_FAIL_STREAK=0 ;; - esac - - case "$ACTION" in - complete) - cw_log "Role complete; finalising via egg-orch consensus confirmed." - timeout 30 egg-orch consensus confirmed >/dev/null 2>&1 || true - cw_log "Exiting (role complete)." - exit 0 - ;; - confirm) - # Reviewer §1 (slice-4-blocking): ``note_progress`` must only - # fire when the CLI actually succeeded. Otherwise a persistent - # 5xx / transport / ``producer_not_fully_acked`` race against - # ``egg-orch consensus confirmed`` becomes a tight retry loop - # (~tens of ms per iteration, two short HTTP calls) that - # silently drains budget because the idle latch keeps resetting. - # The legacy template guarded this with a 3-restart cap; the - # event-pump path's equivalent is the idle-budget safety net - # gated on rc. - cw_log "Confirming via egg-orch consensus confirmed." - timeout 30 egg-orch consensus confirmed >/dev/null 2>&1 - confirm_rc=$? - if [ "$confirm_rc" -eq 0 ]; then - note_progress - CONFIRM_FAIL_STREAK=0 - else - # Floor the retry cadence so a persistent failure can't - # hot-loop the orchestrator faster than the idle counter - # can age. The sleep grows linearly with the streak length - # (capped at 30 s) so the operator sees the idle alert - # within ~30 min on the default budget while consecutive - # short failures don't escalate the load on the route. - CONFIRM_FAIL_STREAK=$(( CONFIRM_FAIL_STREAK + 1 )) - # NOTE: bash ``case``-branch scope is global, not function- - # local. Name reflects scope to avoid the false suggestion - # that this could be ``declare local`` (reviewer §4). - confirm_backoff_secs=$(( CONFIRM_FAIL_STREAK * 2 )) - if [ "$confirm_backoff_secs" -gt 30 ]; then - confirm_backoff_secs=30 - fi - cw_log "consensus confirmed failed (rc=$confirm_rc, streak=$CONFIRM_FAIL_STREAK); backing off ${confirm_backoff_secs}s. Idle counter continues to accrue." - sleep "$confirm_backoff_secs" - fi - ;; - wait) - # Block on the bus. ``wait_for_event`` returns 0 only when - # a matching message was delivered (sandbox/egg_lib CLI - # contract: ``message wait-loop`` exits 0 on match, 1 on - # safety-cap / no-match). Reset the idle counter ONLY in - # the match case -- a timeout return is the DEFINITION of - # idle, not progress. (reviewer_concurrency v1 finding 2 / - # #2908 slice-2 NACK: unconditional ``note_progress`` here - # would defeat the entire idle-budget safety net because - # the inner wait-loop returns every ~60 s with no event.) - wait_for_event "$ROLE_CONFIRMED" - wait_rc=$? - if [ "$wait_rc" -eq 0 ]; then - note_progress - fi - ;; - propose|ack|nack) - # Reviewer §1 (slice-4-blocking): symmetric with the ``confirm`` - # arm above -- ``note_progress`` must only fire when the agent - # invocation actually succeeded. A persistent ``mcp__brc__propose`` - # / API-quota / prompt-rendering failure can fail in well under a - # second, and without rc-gating here the idle latch resets every - # iteration so the operator-visible idle alert never fires. The - # PR removed the legacy 3-restart cap; this rc gate is the - # equivalent ceiling on the action path. - cw_log "Invoking agent (action=$ACTION)." - # R11a (sync_to_proposals docstring anchor): the ``propose`` arm - # intentionally falls through this gate without syncing -- a - # producer's own commits are already on HEAD in its worktree, - # and merging peer proposal commits onto a producer turn is - # exactly the dual-role bleed-through the sync is designed to - # avoid. Only ``ack`` / ``nack`` (reviewer arms) sync. - if [ "$ACTION" = "ack" ] || [ "$ACTION" = "nack" ]; then - sync_to_proposals "$EVENT_PAYLOAD" - fi - agent_invoke_start=$SECONDS - invoke_agent_for_event "$ACTION" "$EVENT_PAYLOAD" - agent_rc=$? - agent_invoke_secs=$(( SECONDS - agent_invoke_start )) - if [ "$agent_rc" -eq 0 ]; then - note_progress - AGENT_FAIL_STREAK=0 - else - AGENT_FAIL_STREAK=$(( AGENT_FAIL_STREAK + 1 )) - cw_log "agent invocation failed (action=$ACTION, rc=$agent_rc, streak=$AGENT_FAIL_STREAK, duration=${agent_invoke_secs}s). Idle counter continues to accrue." - if [ "$AGENT_FAIL_STREAK" -ge 5 ] && [ "$AGENT_FAIL_ALERTED_5" != "true" ]; then - # The per-failure line above already printed the streak - # count this iteration; this escalation line adds the - # diagnosis (likely permanent), not the count. - cw_log "agent invocation streak crossed 5 (action=$ACTION) -- this is likely a permanent failure (unknown model alias, auth misconfiguration, prompt-rendering crash), not a transient. Idle budget continues to accrue." - AGENT_FAIL_ALERTED_5=true - fi - if [ "$AGENT_FAIL_STREAK" -ge 10 ] && [ "$AGENT_FAIL_ALERTED_10" != "true" ]; then - raise_agent_fail_alert "$AGENT_FAIL_STREAK" "$agent_rc" "$agent_invoke_secs" "$ACTION" - AGENT_FAIL_ALERTED_10=true - fi - # #3138: the old flat ``sleep 1`` here assumed agent - # startup gives a natural seconds-to-tens-of-seconds - # floor, but a pre-SDK-init failure (unknown model - # alias, prompt-rendering crash) fails in <1 s and was - # retried at the floor indefinitely (160+ consecutive - # retries on the first issue-3077 run). Backoff parity - # with the ``confirm`` arm above: linear in the streak, - # capped at 30 s, so a deterministic fast-fail loop - # stops hammering the orchestrator while a one-off - # transient still retries promptly. - agent_backoff_secs=$(( AGENT_FAIL_STREAK * 2 )) - if [ "$agent_backoff_secs" -gt 30 ]; then - agent_backoff_secs=30 - fi - sleep "$agent_backoff_secs" - fi - ;; - *) - # Defensive: unknown action surfaced from the orchestrator - # (older orchestrator, schema drift, transient error). Short - # sleep + recheck rather than a tight loop. The idle counter - # still ticks against the configured budget. - cw_log "Unknown next-action='$ACTION'; sleeping briefly and re-fetching." - sleep 5 - ;; - esac - - check_idle_budget -done diff --git a/orchestrator/tests/test_concurrent_executor.py b/orchestrator/tests/test_concurrent_executor.py index 3880f791ad..3e0d3b393a 100644 --- a/orchestrator/tests/test_concurrent_executor.py +++ b/orchestrator/tests/test_concurrent_executor.py @@ -282,7 +282,14 @@ def test_get_agent_roles_without_override_uses_defaults(self): # Without override, should return all default implement roles assert len(roles) > 2 - def test_spawn_all_with_roles_override_only_spawns_overridden_roles(self): + def test_spawn_all_with_roles_override_only_registers_overridden_roles(self): + """The ``roles`` override scopes which roles the event loop covers. + + Post-#3164 ``spawn_all`` no longer fans out up-front pods (the + orchestrator event loop spawns one-shot per event), so the override's + effect is on which roles are *registered* with the tracker — exactly + the cohort the loop derives against. No up-front ``spawn_fn`` calls. + """ from concurrent_executor import ConcurrentPhaseExecutor from egg_orchestrator.types import AgentRole @@ -293,22 +300,27 @@ def test_spawn_all_with_roles_override_only_spawns_overridden_roles(self): override = [AgentRole.CODER, AgentRole.REVIEWER_CODE] executor = ConcurrentPhaseExecutor(pipeline, spawn_fn=mock_spawn, roles=override) - with ( - patch("concurrent_executor.create_peer_consensus_tracker") as mock_tracker, - patch("concurrent_executor.emit_event"), - ): - mock_tracker_instance = MagicMock() - mock_tracker.return_value = mock_tracker_instance + try: + with ( + patch("concurrent_executor.create_peer_consensus_tracker") as mock_tracker, + patch("concurrent_executor.emit_event"), + ): + mock_tracker_instance = MagicMock() + mock_tracker.return_value = mock_tracker_instance - executor.spawn_all() + result = executor.spawn_all() + finally: + executor.stop_event_loop() - # Only 2 agents should be spawned and registered (coder + reviewer_code) - assert mock_spawn.call_count == 2 - assert mock_tracker_instance.register_agent.call_count == 2 - registered_roles = [ - call.args[0] for call in mock_tracker_instance.register_agent.call_args_list - ] - assert set(registered_roles) == {"coder", "reviewer_code"} + # No up-front pods; the event loop owns spawning. + assert result == [] + assert mock_spawn.call_count == 0 + # Only the 2 overridden roles are registered with the tracker. + assert mock_tracker_instance.register_agent.call_count == 2 + registered_roles = [ + call.args[0] for call in mock_tracker_instance.register_agent.call_args_list + ] + assert set(registered_roles) == {"coder", "reviewer_code"} class TestIsTransientAgentError: @@ -1461,20 +1473,12 @@ def test_spawn_with_missing_repositories_yaml_does_not_raise(self): class TestEventLoopOwnershipSpawnGating: - """Slice-2 (#3064 TASK-2-3): EGG_EVENT_LOOP_OWNER gates up-front spawning. - - Test-first contract for TASK-2-1's hook into the concurrent executor: - - * ``pod`` (default): ``spawn_all`` fans out exactly as today — one - ``spawn_fn`` call per agent role. Existing pod-mode tests stay green - because this path is untouched. - * ``orchestrator``: ``spawn_all`` spawns NO up-front pods (the event - loop owns the lifecycle and spawns one-shot Jobs per derived event - from the completion-poll site instead). + """#3164: the orchestrator unconditionally owns up-front spawning. - The executor is expected to consult ``env_config.get_event_loop_owner()`` - at ``spawn_all`` time. These tests fail until the coder's parallel hook - lands; convergence makes them green. + ``spawn_all`` spawns NO up-front pods — the orchestrator event loop owns + the lifecycle and spawns one-shot Jobs per derived event. The + ``EGG_EVENT_LOOP_OWNER`` ownership flag (and the pod-mode fan-out it used + to gate) was retired in #3164, so there is a single behavior here. """ def _executor(self, mock_spawn): @@ -1483,48 +1487,15 @@ def _executor(self, mock_spawn): pipeline = _make_pipeline() return ConcurrentPhaseExecutor(pipeline, spawn_fn=mock_spawn) - def test_pod_default_fans_out_one_spawn_per_role(self, monkeypatch): - """Flag unset ⇒ pod mode ⇒ spawn_all fans out every role up front.""" - monkeypatch.delenv("EGG_EVENT_LOOP_OWNER", raising=False) - mock_spawn = MagicMock(return_value=MagicMock(container_id="c")) - executor = self._executor(mock_spawn) - - with ( - patch("concurrent_executor.create_peer_consensus_tracker") as mock_tracker, - patch("concurrent_executor.emit_event"), - ): - mock_tracker.return_value = MagicMock() - executor.spawn_all() - - assert mock_spawn.call_count == len(executor.get_agent_roles()), ( - "pod-default spawn_all must spawn every role up front (unchanged)" - ) - - def test_pod_explicit_fans_out_one_spawn_per_role(self, monkeypatch): - """Explicit EGG_EVENT_LOOP_OWNER=pod behaves identically to unset.""" - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "pod") - mock_spawn = MagicMock(return_value=MagicMock(container_id="c")) - executor = self._executor(mock_spawn) - - with ( - patch("concurrent_executor.create_peer_consensus_tracker") as mock_tracker, - patch("concurrent_executor.emit_event"), - ): - mock_tracker.return_value = MagicMock() - executor.spawn_all() - - assert mock_spawn.call_count == len(executor.get_agent_roles()) - - def test_orchestrator_mode_spawns_no_up_front_pods(self, monkeypatch): - """EGG_EVENT_LOOP_OWNER=orchestrator ⇒ spawn_all spawns no pods up - front; the event loop drives per-event spawns instead. + def test_spawn_all_spawns_no_up_front_pods(self): + """``spawn_all`` spawns no pods up front; the event loop drives + per-event spawns instead. Asserts the loop was actually started (``owns_event_loop()`` / ``_event_loop``) rather than relying on the daemon thread's first-poll sleep to mask premature behavior, and tears the loop down deterministically so the test leaves no live thread behind. """ - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "orchestrator") mock_spawn = MagicMock(return_value=MagicMock(container_id="c")) executor = self._executor(mock_spawn) @@ -1540,13 +1511,9 @@ def test_orchestrator_mode_spawns_no_up_front_pods(self, monkeypatch): # depend on the poll-interval timing window. executor.stop_event_loop() - assert mock_spawn.call_count == 0, ( - "orchestrator mode must not spawn any up-front agent pods" - ) - assert result == [], "orchestrator mode spawn_all returns no executions" - assert executor.owns_event_loop(), ( - "orchestrator mode must start the orchestrator-owned event loop" - ) + assert mock_spawn.call_count == 0, "must not spawn any up-front agent pods" + assert result == [], "spawn_all returns no executions" + assert executor.owns_event_loop(), "must start the orchestrator-owned event loop" assert executor._event_loop is not None diff --git a/orchestrator/tests/test_concurrent_integration.py b/orchestrator/tests/test_concurrent_integration.py index 2ac0255e03..4581623f73 100644 --- a/orchestrator/tests/test_concurrent_integration.py +++ b/orchestrator/tests/test_concurrent_integration.py @@ -680,13 +680,15 @@ def test_spawn_agent_uses_wrapped_command(self): assert command[0] == "bash" assert command[1] == "-c" assert "egg_agent" in command[2] - # Slice-4 (#2908) replaced the capped-restart wrapper with the - # event-pump template; check for its deterministic-loop markers - # instead of the deleted RESTART_COUNT / "BRC Consensus Recovery" - # strings. + # #3164 retired the in-pod wait loop: the wrapper is now a one-shot + # event handler. Check for its markers — it re-derives next-action + # once as a stale-event backstop (``egg-orch brc next-action``) and + # keys on the injected ``EGG_EVENT_ACTION`` — rather than the deleted + # blocking-loop ``egg-orch brc get-state`` / wait-loop strings. assert "event-pump" in command[2] - assert "egg-orch brc get-state" in command[2] assert "egg-orch brc next-action" in command[2] + assert "EGG_EVENT_ACTION" in command[2] + assert "egg-orch brc get-state" not in command[2] class TestNoImplicitReadyOnCleanExit: diff --git a/orchestrator/tests/test_consensus_wrapper.py b/orchestrator/tests/test_consensus_wrapper.py index f348213ff9..8832107bcf 100644 --- a/orchestrator/tests/test_consensus_wrapper.py +++ b/orchestrator/tests/test_consensus_wrapper.py @@ -1,13 +1,24 @@ """Tests for the consensus wrapper module. -Slice-4 task-4-2 deleted the legacy capped-restart template; the -``TestBuildConsensusWrappedCommand`` / ``TestConsensusWrapperBehavior`` -/ ``TestBufferOverflowDetection`` / ``TestEventDrivenWait`` / -``TestSSESigtermGrace`` classes that pinned its surface went with it, -along with the ``_force_legacy_template`` fixture they shared. The -buffer-overflow / transient-crash / startup-failure shell classifier -helpers were preserved (relocated into ``_EVENT_PUMP_WRAPPER_TEMPLATE``); -their coverage is folded into the event-pump test classes below. +Issue #3164 retired the in-pod BRC event-loop "wait arm": the wrapper +rendered by ``build_consensus_wrapped_command`` (and its delegate +``build_event_pump_wrapped_command``) is now ONE-SHOT ONLY. The +orchestrator spawns the wrapper once per actionable BRC event with +``EGG_EVENT_ACTION`` injected; the wrapper handles that single event and +exits. There is no blocking ``while true`` wait-loop, no background +heartbeat emitter, no idle-budget machinery, and no agent-fail-streak +machinery in the wrapper — those were either deleted or re-homed to the +orchestrator event loop. + +The test classes that pinned the deleted in-pod-pump surface (the +``while true`` loop, ``egg-orch message wait-loop``, the 30s background +heartbeat, the idle-budget overseer alert, the wait-filter construction, +the role-complete/consensus-confirmed loop arm, the agent-fail-streak +escalation, the ownership-flag accessor) went with it. What survives: +the shared helper functions (``cw_log`` / ``emit_heartbeat`` / +``fetch_next_action`` / ``sync_to_proposals`` / ``invoke_agent_for_event``), +the one-shot event handler, the per-event prompt composer wiring, the +sync-to-proposal banner contract, and the ``--effort`` thread-through. """ import os @@ -15,1371 +26,8 @@ import subprocess import sys -import pytest from consensus_wrapper import build_consensus_wrapped_command -# Sentinel event types the event-pump wait filter must always cover. -# Plan TASK-2-1 line 797-799 enumerates these six explicitly. -_EXPECTED_EVENT_PUMP_WAIT_FILTERS = ( - "CONSENSUS_PROPOSE", - "CONSENSUS_ACK", - "CONSENSUS_NACK", - "STATUS", - "CONSENSUS_RE_REVIEW", - "OVERSEER_ALERT", -) - - -class TestEventPumpTemplateSelection: - """(i) Template selection. Post slice-4 task-4-2 there is only one - template — the event-pump. The ``EGG_BRC_EVENT_PUMP`` env flag is - no longer read; any value (including ``false`` / ``0`` / ``no`` / - ``off``) is silently inert. The class survives task-4-2 so the - snapshot regression on the event-pump template + the wait-filter - composition keeps a dedicated home. - """ - - def test_flag_unset_emits_event_pump_template_by_default(self, monkeypatch): - """Slice-4 task-4-1 flipped the unset-env default to event-pump; - slice-4 task-4-2 then deleted the legacy template entirely. The - event-pump is the only production path; unset env must emit it. - """ - monkeypatch.delenv("EGG_BRC_EVENT_PUMP", raising=False) - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Event-pump template markers. - assert "Event-pump wrapper (#2908 slice-2)" in script - # Legacy markers MUST NOT appear — they were deleted by task-4-2. - assert "MAX_RESTARTS=" not in script - assert "BRC Consensus Recovery" not in script - - def test_flag_false_is_silently_inert_after_task_4_2(self, monkeypatch): - """Slice-4 task-4-2 deleted the legacy template and the - ``EGG_BRC_EVENT_PUMP`` env-flag read along with it. Any value - (truthy or falsy) is silently ignored; operators with the var - lingering in k8s manifests can leave it set to ``false`` and - still get the event-pump template. - - Pins the inertness so a future regression that re-introduces - a legacy-template branch trips this test. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "false") - cmd_false = build_consensus_wrapped_command("Prompt") - monkeypatch.delenv("EGG_BRC_EVENT_PUMP", raising=False) - cmd_unset = build_consensus_wrapped_command("Prompt") - assert cmd_false[2] == cmd_unset[2], ( - "EGG_BRC_EVENT_PUMP=false must be silently inert post " - "slice-4 task-4-2 — both must emit the event-pump template." - ) - # Same check for the other falsy tokens. - for falsy in ("0", "no", "off", "False", "OFF"): - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", falsy) - assert build_consensus_wrapped_command("Prompt")[2] == cmd_unset[2], ( - f"EGG_BRC_EVENT_PUMP={falsy!r} must be silently inert." - ) - - def test_flag_unset_and_flag_true_emit_identical_scripts(self, monkeypatch): - """Sanity: unset and explicit-true cases emit the same script - post task-4-1. Pinned so a future regression that splits them - apart trips the test rather than silently bifurcating the - production path. - """ - monkeypatch.delenv("EGG_BRC_EVENT_PUMP", raising=False) - cmd_unset = build_consensus_wrapped_command("Prompt") - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd_true = build_consensus_wrapped_command("Prompt") - assert cmd_unset[2] == cmd_true[2], ( - "Unset env and explicit EGG_BRC_EVENT_PUMP=true must emit " - "the same script after the task-4-1 default flip — " - "otherwise the production path is bifurcated." - ) - - def test_flag_on_emits_event_pump_template(self, monkeypatch): - """With ``EGG_BRC_EVENT_PUMP=true``, the new event-pump template - is emitted in place of the legacy capped-restart template. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The new template MUST contain a wait-loop primitive over the - # six event types and handle the ``role_complete`` signal from - # ``brc next-action`` (plan line 783-792). - assert "egg-orch message wait-loop" in script - # The role_complete signal arrives from ``brc next-action`` as the - # action value ``complete``. The wrapper must branch on it (case - # arm or equivalent). Accept any of: literal ``role_complete`` - # token (variable name), the ``complete)`` case arm in the action - # switch, or a ``ROLE_CONFIRMED`` boolean derived from - # ``brc get-state``. - assert any( - marker in script for marker in ("role_complete", "complete)", "ROLE_CONFIRMED") - ), ( - "event-pump must check role_complete from brc get-state / " - "next-action (plan line 783-792); neither role_complete nor " - "ROLE_CONFIRMED nor a complete) case arm found in script." - ) - # New template must call ``brc next-action`` (plan line 785). - assert "brc next-action" in script - - def test_flag_on_wait_filter_contains_six_required_events(self, monkeypatch): - """(i) The flag-on path's wait-filter set must include all six event - types the plan enumerates (line 797-799). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - for event_type in _EXPECTED_EVENT_PUMP_WAIT_FILTERS: - assert f"--for {event_type}" in script, ( - f"event-pump wait filter missing --for {event_type}; " - f"plan TASK-2-1 line 797-799 requires all six." - ) - - -class TestEventPumpHeartbeatCadence: - """(ii)+(iii) Wrapper-side heartbeat emission migrated out of - ``sandbox/egg_agent_tools/handlers/message.py:267-429`` and into the - event-pump bash. The payload must include ``slice_id`` sourced from - ``EGG_SLICE_ID`` so a regression in slice_id propagation (risk_analyst - R9) is caught directly. - - Cadence is verified with a mock subprocess fast-forward — we inspect - the generated script for the configured 30-second loop interval and - the existence of a backgrounded heartbeat subshell, rather than - sleeping in real wall-clock to keep the test deterministic. - """ - - def test_flag_on_emits_heartbeat_subshell(self, monkeypatch): - """The event-pump template must contain ``egg-orch message - heartbeat`` invoked in a backgrounded subshell while the - wait-loop is blocking (plan TASK-2-2 description, line 828-829). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - assert "egg-orch message heartbeat" in script - # Background subshell shape: `( ... ) &` or a backgrounded - # subshell variant — flexible enough to match either. - # Heartbeat must run alongside the wait-loop, not block it. - # We pin the presence of the heartbeat command + the ``wait-loop`` - # primitive in the same script so the migration cannot regress - # to agent-side-only heartbeating. - assert "wait-loop" in script - - def test_flag_on_heartbeat_cadence_is_30_seconds(self, monkeypatch): - """The plan (TASK-2-2 description, line 828) names a 30-second - cadence. Pin the default in the script so a regression to a - different cadence is caught by this test. Accept either a - literal ``sleep 30`` or an env-var indirection that defaults to - 30 (e.g. ``${EGG_BRC_HEARTBEAT_INTERVAL_SECS:-30}``). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Accept either form: literal ``sleep 30`` or an env-var - # default of 30 (``:-30}"`` etc.). - cadence_markers = ( - "sleep 30", - ":-30}", - "INTERVAL_SECS=30", - "INTERVAL_SECS:-30", - ) - assert any(m in script for m in cadence_markers), ( - "wrapper-side heartbeat must default to a 30s cadence per " - "plan TASK-2-2 line 828; neither literal `sleep 30` nor an " - "env-var default of 30 found in the rendered bash." - ) - - def test_flag_on_heartbeat_payload_threads_slice_id_from_env(self, monkeypatch): - """(iii) The heartbeat payload MUST source ``slice_id`` from the - ``EGG_SLICE_ID`` env var. Plan TASK-2-2 line 831-834 names this - invariant directly: "The heartbeat payload MUST include - ``slice_id == os.environ['EGG_SLICE_ID']`` (or the equivalent - shell substitution ``${EGG_SLICE_ID:-}`` passed through the CLI) - so a regression in slice_id propagation is caught directly." - - We assert the script references ``EGG_SLICE_ID`` adjacent to the - ``egg-orch message heartbeat`` invocation. A shell substitution - of the env var into the heartbeat command line satisfies the - "passed through the CLI" route from the plan. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The script must reference EGG_SLICE_ID; the message heartbeat - # CLI takes ``--slice-id`` or threads it through the request. - assert "EGG_SLICE_ID" in script, ( - "heartbeat payload must source slice_id from EGG_SLICE_ID; " - "regression in slice_id propagation will not be caught " - "without this wiring (risk_analyst R9)." - ) - - # Note: slice-2's ``test_flag_off_heartbeat_path_unchanged`` was - # deleted by slice-4 task-4-2. The legacy template (and its - # "no wrapper-side heartbeat under flag-off" invariant) no longer - # exists; the agent-side ``handlers/message.py:_default_emit_wait_loop_heartbeat`` - # was deleted alongside the legacy template, so the double-heartbeat - # bus-spam scenario that test guarded against is also gone. The - # wrapper now unconditionally emits the wrapper-owned heartbeat - # subshell (see ``test_flag_on_emits_heartbeat_subshell`` / - # ``test_flag_on_emits_heartbeat_subshell_template_marker`` for - # the post-deletion invariant). - - -class TestEventPumpKeepAliveCadence: - """(iv) Wrapper-side gateway-session keep-alive (#2451) migrated out - of ``sandbox/egg_agent_tools/handlers/message.py`` and into the - event-pump bash. The wrapper performs the same lifecycle-secret-gated - session refresh as a background subshell alongside the heartbeat - emitter from TASK-2-2. - """ - - def test_flag_on_emits_keep_alive_subshell(self, monkeypatch): - """The event-pump template must perform a gateway-session - refresh while the wait-loop is blocking (plan TASK-2-4 line - 875-881). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The keep-alive ping refreshes the lifecycle-secret-gated - # session. Either an explicit ``keep-alive``/``keepalive`` - # subcommand call or a session-refresh marker must appear. - assert "keep-alive" in script or "keepalive" in script or "session" in script.lower(), ( - "event-pump template must perform gateway-session keep-alive " - "(plan TASK-2-4); without it, long waits will lose their " - "lifecycle-secret-gated session and the next CLI call will " - "401." - ) - - # Note: slice-2's ``test_flag_off_keep_alive_remains_agent_side`` was - # deleted by slice-4 task-4-2. The legacy template is gone and the - # agent-side keep-alive (which lived inside ``message_wait_loop``'s - # heartbeat path) was deleted along with the legacy template, so - # the "old path unchanged" invariant no longer applies. - - -class TestEventPumpIdleBudgetAlert: - """(v) Idle / no-progress safety budget driven by env - ``EGG_BRC_IDLE_BUDGET_MIN`` (default 30). When no actionable event - has arrived for the budget duration, the wrapper emits - ``mcp__progress__overseer_alert`` (anomaly - ``stuck-phase-transition``, priority ``high``) and continues - blocking. The legacy template that owned the historical restart - cap was deleted in slice-4 task-4-2; the idle budget is now the - only liveness ceiling in the wrapper. - - NOTE: Per scope update on #2908 issue body and contract cq-3, the - durable server-side ``Pipeline.no_progress_budget`` is the binding - primary mechanism. The in-wrapper env-var budget tested here is the - slice-2 implementation gate; it must work AND must NOT replace the - durable server-side path (which lands in slice-1's orchestrator - route work, not here). - """ - - def test_flag_on_contains_idle_budget_alert(self, monkeypatch): - """Idle budget threshold triggers an overseer alert at the - configured duration (plan TASK-2-3 acceptance line 863-868). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Default budget: 30 minutes (plan line 854). - assert "EGG_BRC_IDLE_BUDGET_MIN" in script - # Alert payload (plan line 857-858) — overseer alert with the - # right anomaly + priority. - assert "overseer alert" in script or "overseer_alert" in script - assert "stuck-phase-transition" in script - # Priority "high" must be passed somewhere (either as - # ``--priority high`` literal, ``--priority "$priority"`` with - # ``"high"`` passed in, or an env-var default of "high"). We - # require both the ``--priority`` flag AND the ``high`` token - # appear in the script. - assert "--priority" in script, ( - "overseer alert payload must include --priority flag (plan " - "TASK-2-3 line 857-858 — `priority high`)." - ) - assert "high" in script, ( - "overseer alert priority must be `high` per plan TASK-2-3 line 857-858." - ) - - def test_flag_on_idle_budget_default_30_minutes(self, monkeypatch): - """Default ``EGG_BRC_IDLE_BUDGET_MIN`` is 30 minutes per plan - line 853-854 (well above the WS7-observed 10-13 min idle ceiling). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Default value must appear as a literal in the rendered bash so - # an operator override flows through. - assert "${EGG_BRC_IDLE_BUDGET_MIN:-30}" in script or "EGG_BRC_IDLE_BUDGET_MIN=30" in script - - def test_flag_on_idle_budget_continues_blocking_after_alert(self, monkeypatch): - """After the alert fires, the loop continues blocking (NOT exit - 1 → FAILED). Plan line 867-868: "loop continues blocking after - alert (not exit 1 → FAILED)." - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The alert dispatch must NOT be immediately followed by an - # ``exit 1``. We assert the pattern by checking that the script - # re-enters the wait-loop after the alert (continue / loop / - # re-block). The simplest pin: there is no immediate ``exit 1`` - # adjacent to the ``stuck-phase-transition`` keyword. - alert_idx = script.find("stuck-phase-transition") - if alert_idx == -1: - # If the test for `test_flag_on_contains_idle_budget_alert` - # already failed, this test would also fail — but its scope - # is the continue-not-exit invariant, so we skip cleanly - # rather than double-report. - import pytest as _pytest - - _pytest.skip( - "stuck-phase-transition keyword absent; covered by the " - "TestEventPumpIdleBudgetAlert.test_flag_on_contains_idle_budget_alert " - "failure" - ) - # Look at the next 200 characters after the alert keyword — no - # adjacent ``exit 1`` must appear (the loop continues). - nearby = script[alert_idx : alert_idx + 200] - assert "exit 1" not in nearby, ( - "idle-budget alert must NOT be followed by exit 1; the loop " - "MUST continue blocking (plan line 867-868)." - ) - - # Note: slice-2's ``test_flag_off_idle_budget_not_used`` was deleted - # by slice-4 task-4-2. The legacy template is gone; the unset-env - # path now emits the event-pump template, which DOES reference - # ``EGG_BRC_IDLE_BUDGET_MIN``. The "legacy template keeps the - # 3-restart cap verbatim" invariant no longer applies — see - # ``test_idle_budget_default_30_min_in_script`` for the post-deletion - # invariant. - - -class TestEventPumpStaleVersionRefetch: - """(vi) 409 ``stale_version`` from ``brc next-action`` is an - event-pump signal (re-fetch state, re-invoke), NOT a transient crash - to retry with backoff. - - Plan TASK-2-1 line 793-796: "Wrapper handles 409 ``stale_version`` - and 409 aggregated-NACK from ``brc next-action`` as event-pump - signals (re-fetch state, re-invoke), NOT as transient crashes to - retry with backoff." - """ - - def test_flag_on_handles_409_stale_version_as_refetch(self, monkeypatch): - """The event-pump bash MUST treat HTTP 409 from ``brc - next-action`` as a re-fetch trigger — call ``brc get-state`` - again and re-invoke, NOT retry the same call with backoff. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The script must explicitly match the 409 status code and - # call ``brc get-state`` again. Pin the literal "409" so a - # blanket "any error → retry" path is caught by this test. - assert "409" in script - # Re-fetch primitive must be present. - assert "brc get-state" in script - - def test_flag_on_409_does_not_apply_backoff(self, monkeypatch): - """On 409, the wrapper must NOT enter the ``CRASH_BACKOFF`` / - sleep-then-retry path. This is the negative invariant of (vi). - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The event-pump template should not be using CRASH_BACKOFF - # variables at all (those belong to the legacy template). - # If they DO leak in via copy-paste, the 409 handler might - # accidentally land in the wrong branch. - # Allow CRASH_BACKOFF only if it is decisively scoped away - # from the 409 handler. - idx = script.find("409") - if idx != -1: - nearby = script[max(0, idx - 200) : idx + 400] - assert "CRASH_BACKOFF" not in nearby, ( - "409 stale_version handler must not be co-located with " - "CRASH_BACKOFF backoff — it is a re-fetch signal, NOT a " - "transient crash (plan TASK-2-1 line 793-796)." - ) - - -class TestEventPumpRoleCompleteConfirm: - """(vii) + (vii.b) ``role_complete=true`` path calls ``egg-orch - consensus confirmed`` and exits 0; the wrapper does NOT also call - ``egg-orch progress complete`` (defensive guard against the - pseudocode-typo the architect corrected — plan line 932-934). - """ - - def test_flag_on_role_complete_calls_consensus_confirmed(self, monkeypatch): - """On ``role_complete=true`` the event-pump bash calls - ``egg-orch consensus confirmed`` to mark consensus and exits 0. - Plan TASK-2-1 line 791-793: "the wrapper calls ``egg-orch - consensus confirmed`` (existing CLI at orch_cli.py:2753) — NOT a - new ``progress complete`` command — to mark the role's consensus - and exit 0." - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - assert "egg-orch consensus confirmed" in script - # Clean exit 0 must appear in the role_complete branch. - # We check by locating the consensus-confirmed call and asserting - # an ``exit 0`` follows somewhere in the rest of that branch. - idx = script.find("egg-orch consensus confirmed") - assert idx >= 0 - tail = script[idx:] - assert "exit 0" in tail, ( - "the role_complete branch that calls `egg-orch consensus " - "confirmed` must exit 0 (plan TASK-2-1 line 791-793)." - ) - - def test_flag_on_does_not_call_progress_complete(self, monkeypatch): - """(vii.b) Defensive guard: the wrapper template must NOT contain - ``progress complete`` — that would be the pseudocode-typo the - architect corrected and is NOT a valid CLI subcommand for marking - BRC consensus. - - Plan TASK-2-6 acceptance line 949-950: "test (vii.b) asserts - ``rg 'progress complete'`` against the emitted bash returns - zero matches." - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - assert "progress complete" not in script, ( - "wrapper must NOT call `egg-orch progress complete` — the " - "correct CLI is `egg-orch consensus confirmed`. This guard " - "catches the pseudocode-typo the architect corrected." - ) - - # Note: slice-2's ``test_flag_off_legacy_path_does_not_auto_call_consensus_confirmed`` - # was deleted by slice-4 task-4-2. The legacy template is gone; the - # event-pump template DOES invoke ``egg-orch consensus confirmed`` - # under the ``confirm`` / ``complete`` arms of the action loop (driven - # by ``brc next-action``, not auto-invoked on agent exit). The - # symmetry-guard concern (the wrapper auto-confirming on behalf of - # the agent) is now structurally impossible — the only invocations - # happen inside the ``case "$ACTION"`` arms which require an - # orchestrator-side derivation, not a wrapper-side timer or exit - # condition. - - -class TestEventPumpWaitFilterConditional: - """(viii) Wait-filter construction OMITS ``CONSENSUS_CONFIRMED`` - pre-confirm and INCLUDES it post-confirm. - - Plan TASK-2-1 line 811-816: "the wait-filter set is **constructed - conditionally from ``consensus_status.is_role_confirmed``** — - pre-confirm waits OMIT ``CONSENSUS_CONFIRMED`` from the filter (per - risk_analyst R12 / orchestrator HTTP-400 rejection documented in - #2064/#2482), post-confirm STAY-ALIVE waits INCLUDE it." - - The HTTP-400 rejection is real: the orchestrator's wait endpoint - returns 400 if a producer's pre-confirm wait names - ``CONSENSUS_CONFIRMED`` because its own confirm is what generates - that signal. So the wrapper bash MUST conditionally include the - filter or risk wedging every pre-confirm wait. - """ - - def test_flag_on_wait_filter_is_constructed_conditionally(self, monkeypatch): - """The event-pump bash MUST branch on ``is_role_confirmed`` (or - an equivalent boolean) when constructing the wait-loop filter - set so the pre-confirm path omits ``CONSENSUS_CONFIRMED``. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The conditional may take any of several shapes. The plan names - # ``consensus_status.is_role_confirmed`` as the input — at the - # bash level this is a boolean variable derived from the - # ``brc get-state`` response. Any of these markers proves the - # conditional shape exists: - markers = ( - "is_role_confirmed", - "ROLE_CONFIRMED", - "is_confirmed", - "role_confirmed", - ) - assert any(m in script for m in markers), ( - "event-pump wait filter must be constructed conditionally " - "from a role-confirmed boolean derived from brc get-state; " - "pre-confirm waits must omit CONSENSUS_CONFIRMED per " - "risk_analyst R12 / orchestrator HTTP-400 rejection " - "(#2064/#2482)." - ) - - def test_flag_on_pre_confirm_wait_does_not_always_include_consensus_confirmed( - self, monkeypatch - ): - """Negative invariant: the script must NOT unconditionally pass - ``--for CONSENSUS_CONFIRMED`` to the wait-loop. If every - wait-loop invocation hard-codes that filter, the conditional - shape is missing and pre-confirm waits will wedge with HTTP 400. - - The matching tactic: ensure that ``CONSENSUS_CONFIRMED`` does - NOT appear in the same line as ``wait-loop`` unconditionally. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Find every line containing ``wait-loop``. If any of them - # ALSO contains ``--for CONSENSUS_CONFIRMED`` directly, - # without a conditional gate, that's a regression. - # We detect this by checking that the script branches on a - # role-confirmed boolean somewhere near the wait-loop call. - wait_loop_lines = [ - ln for ln in script.splitlines() if "wait-loop" in ln and "egg-orch" in ln - ] - # Allow at most one wait-loop call site, but require the script - # contains either an ``if`` branch around it or a variable that - # is conditionally extended. - if any("CONSENSUS_CONFIRMED" in ln for ln in wait_loop_lines): - # The literal --for CONSENSUS_CONFIRMED appears on the same - # line as wait-loop. This is only OK if the line is gated - # by a conditional; check for a same-region ``if``/``case`` - # construct. - assert "if " in script and ( - "is_role_confirmed" in script - or "ROLE_CONFIRMED" in script - or "is_confirmed" in script - ), ( - "the wait-loop invocation includes --for " - "CONSENSUS_CONFIRMED unconditionally; this will wedge " - "pre-confirm waits with HTTP 400 (risk_analyst R12)." - ) - - def test_flag_on_post_confirm_wait_includes_consensus_confirmed(self, monkeypatch): - """Positive invariant: post-confirm STAY-ALIVE waits MUST - include ``CONSENSUS_CONFIRMED`` so the wrapper wakes when peer - producers confirm. Plan line 815-816: "post-confirm STAY-ALIVE - waits INCLUDE it." - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The literal ``--for CONSENSUS_CONFIRMED`` MUST appear somewhere - # in the script (gated by the conditional from the previous - # test). - assert "--for CONSENSUS_CONFIRMED" in script, ( - "post-confirm STAY-ALIVE wait must include " - "--for CONSENSUS_CONFIRMED (plan line 815-816)." - ) - - -class TestEventPumpSliceIdHeartbeatEdge: - """(ix) Unset-``EGG_SLICE_ID`` case (plan / refine phase) emits - either explicit-null or omitted slice_id on the heartbeat payload — - NOT empty-string. - - Plan TASK-2-6 acceptance line 937-939: "unset-``EGG_SLICE_ID`` case - (plan/refine phase) emits either explicit-null or omitted slice_id - on the heartbeat payload (NOT empty-string)." - - Empty-string is a known bug class: the orchestrator's slice scoping - treats "" as a match for "no slice" but also as a distinct value - from None, so a heartbeat with ``slice_id=""`` will mismatch - against a tracker reconstruction keyed on None. This test pins the - null / omission shape. - """ - - def test_unset_slice_id_does_not_emit_empty_string(self, monkeypatch): - """When ``EGG_SLICE_ID`` is unset, the rendered bash must NOT - pass an empty-string ``slice_id`` to the heartbeat CLI. - - We assert by scanning the script for a literal ``--slice-id ""`` - or ``"slice_id":""`` pattern, both of which are bug shapes. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - monkeypatch.delenv("EGG_SLICE_ID", raising=False) - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Neither of these bug patterns may appear in the rendered bash. - assert '--slice-id ""' not in script, ( - 'rendered bash must not emit `--slice-id ""` — plan/refine ' - "phases run without a slice and the heartbeat payload must " - "omit slice_id (or send null), NOT empty-string." - ) - assert '"slice_id":""' not in script - assert '"slice_id": ""' not in script - - def test_slice_id_threaded_via_shell_substitution(self, monkeypatch): - """The plan names two acceptable threading shapes (TASK-2-2 - line 831-834): - - - ``slice_id == os.environ['EGG_SLICE_ID']`` (Python-side read), OR - - ``${EGG_SLICE_ID:-}`` shell substitution passed through the CLI. - - For the bash template the substitution form is the natural - shape. Pin the presence of a ``${EGG_SLICE_ID...}`` substitution - anywhere in the script — the CLI's ``cmd_message_heartbeat`` - already resolves the env var server-side, so even an - empty-string default would be filtered by the handler. But the - rendered bash MUST still reference the env var so the slice - scope makes it onto the wire. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - assert "EGG_SLICE_ID" in script, ( - "rendered bash must reference EGG_SLICE_ID so slice scoping " - "propagates onto the heartbeat payload (risk_analyst R9)." - ) - - -class TestEventPumpIdleBudgetCeiling: - """Post slice-4 task-4-2 the event-pump template is the only - template path and the ``EGG_BRC_EVENT_PUMP`` env flag is silently - inert (the legacy capped-restart template and the env-flag read - were both deleted). This class pins the surviving invariant: the - idle budget — not a restart cap — is the liveness ceiling for the - wrapper. Originally named ``TestEventPumpFlagIsolation`` (slice-2) - when the slice-2/3/4 split between event-pump and legacy paths - needed to be policed; renamed and trimmed in slice-4 task-4-2 to - match the post-deletion single-template world. - """ - - def test_event_pump_relies_on_idle_budget_not_legacy_restart_cap(self, monkeypatch): - """The event-pump path uses ``EGG_BRC_IDLE_BUDGET_MIN`` as the - liveness ceiling — the legacy restart cap was deleted by - slice-4 task-4-2 along with the ``max_restarts`` kwarg on - ``build_consensus_wrapped_command``. - - Renamed and simplified from the original - ``test_flag_on_does_not_inherit_legacy_max_restarts``: the - kwarg is gone and the env flag is silently inert, so the test - no longer needs to drive either. - """ - monkeypatch.delenv("EGG_BRC_EVENT_PUMP", raising=False) - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - assert "EGG_BRC_IDLE_BUDGET_MIN" in script, ( - "event-pump path must rely on EGG_BRC_IDLE_BUDGET_MIN as the " - "liveness ceiling (slice-4 task-4-2 deleted the legacy " - "restart cap)." - ) - - def test_flag_on_does_not_re_invoke_recovery_system_prompt(self, monkeypatch): - """The new template does not need the legacy recovery system - prompt — the per-event invocation contract supplies its own - memory + delta context. Plan slice-3 owns the per-event prompt - composer. - - Pin that the legacy ``BRC Consensus Recovery`` header does NOT - appear in the flag-on template; carrying it forward would - confuse a one-shot per-event invocation with a recovery from - crash. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The legacy recovery header text is a strong marker. - assert "BRC Consensus Recovery" not in script, ( - "flag-on event-pump template must not carry the legacy " - "'BRC Consensus Recovery' header forward — the per-event " - "invocation contract supplies its own context." - ) - - -class TestEventPumpHeartbeatSubshellLifecycle: - """Adversarial probing: the wrapper-owned heartbeat subshell MUST be - killable by ``stop_background_heartbeat``. If the subshell - installs an empty ``trap '' TERM`` then a default ``kill`` (SIGTERM) - from the parent is IGNORED — the subshell continues forever, and - the subsequent ``wait $HB_BG_PID`` blocks indefinitely because the - process never exits. This wedges the entire wrapper loop after the - first ``wait_for_event`` call returns. - - Issue lineage: this is exactly the bug class #2906 / #2451 were - trying to fix at the agent-side layer. Re-introducing it at the - wrapper layer would silently regress slice_id propagation AND wedge - the deterministic loop. - - Note: behavioural verification of the kill semantics belongs in a - bash-harness integration test (where we can spawn a subshell with - the same trap and confirm that ``kill && wait`` does not return). - These tests pin the *static* invariant against the rendered bash: - if the subshell traps TERM, the corresponding ``stop`` path MUST - use a signal that the subshell does not trap (``SIGINT``, - ``SIGHUP``, or ``SIGKILL``); otherwise the lifecycle is broken. - """ - - def test_flag_on_heartbeat_subshell_can_be_stopped(self, monkeypatch): - """If the rendered bash installs ``trap '' TERM`` (or any - ignored-TERM equivalent) in the heartbeat subshell, the - corresponding ``stop`` path MUST send a non-TERM signal so the - subshell actually exits. Sending the default ``kill`` (= TERM) - against a TERM-ignoring trap is a silent no-op — the subshell - loops forever and the wait blocks indefinitely. - - Verified by ad-hoc bash harness during slice-2 review: - - $ bash -c '( trap "" TERM; while true; do sleep 1; echo tick; done ) & - sleep 2; kill $!; wait $! 2>/dev/null' - tick - tick - (hang — never returns) - - Therefore the invariant: if the rendered bash sets - ``trap '' TERM`` in the heartbeat subshell, the stop primitive - must NOT rely on a default-signal ``kill``. Use ``kill -INT``, - ``kill -HUP``, ``kill -KILL``, or do not install the empty - TERM trap in the first place. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - - # Strip ``#``-prefixed comment content so a comment that *mentions* - # the buggy pattern (``# The earlier `trap '' TERM` form ...``) - # doesn't trip the detector. We use a per-line scan rather than - # full bash tokenisation — sufficient for the static invariant. - def _strip_comments(text: str) -> str: - out_lines = [] - for ln in text.splitlines(): - stripped = ln.lstrip() - if stripped.startswith("#"): - continue - # Trim inline trailing comments (best-effort: ``#`` outside - # any quotes). False-positive risk is bounded because the - # pattern we look for has its own quote shape. - if " #" in ln: - ln = ln.split(" #", 1)[0] - out_lines.append(ln) - return "\n".join(out_lines) - - executable = _strip_comments(script) - # Two failure modes the test guards against: - # - # (A) Heartbeat subshell installs `trap '' TERM` AND the - # ``stop`` path issues a default-signal `kill $HB_BG_PID` - # (with no explicit signal). This is the silent-wedge bug. - # - # (B) Heartbeat subshell installs `trap '' TERM` AND the - # ``stop`` path issues `kill -TERM`/`kill -15`. Same wedge, - # different shape. - # - # If the subshell does NOT install an ignored-TERM trap (either - # absent or replaced with ``trap 'exit 0' TERM`` / equivalent - # handler), this test is automatically satisfied — the wrapper is - # free to use any kill primitive against a non-trapping (or - # cleanly-exiting) subshell. - installs_ignoring_term_trap = "trap '' TERM" in executable or 'trap "" TERM' in executable - if not installs_ignoring_term_trap: - # The subshell either does not trap TERM at all, or installs a - # handler that exits cleanly on TERM (e.g. ``trap 'exit 0' - # TERM``). Either way the default-signal kill / wait pair - # works as expected. - return - # The subshell installs an ignored-TERM trap. The stop path MUST - # use a non-TERM signal. Allowed primitives: ``kill -INT``, - # ``kill -HUP``, ``kill -KILL``, ``kill -9``, or - # ``kill -SIGINT``/-SIGHUP/-SIGKILL. Scan the script for any of - # these adjacent to the ``HB_BG_PID`` symbol. - allowed_stop_primitives = ( - "kill -INT", - "kill -HUP", - "kill -KILL", - "kill -9", - "kill -SIGINT", - "kill -SIGHUP", - "kill -SIGKILL", - ) - kill_lines = [ln for ln in executable.splitlines() if "kill " in ln and "HB_BG_PID" in ln] - # Detect a default-signal kill (no explicit -SIG flag). - default_kill_lines = [ - ln - for ln in kill_lines - if not any(prim in ln for prim in allowed_stop_primitives) and "kill -" not in ln - ] - assert not default_kill_lines, ( - "the heartbeat subshell installs `trap '' TERM` which " - "ignores the default SIGTERM signal. The corresponding " - "stop path uses a default-signal `kill` which is a silent " - "no-op against that trap — the subshell will never exit, " - "and the subsequent `wait` blocks indefinitely, wedging " - "the event-pump loop after the first wait_for_event call.\n" - "Fix options: (a) remove the `trap '' TERM` from the " - "subshell, (b) replace it with `trap 'exit 0' TERM` (or any " - "handler that exits the subshell), or (c) change the stop " - "path to `kill -INT`, `kill -HUP`, or `kill -KILL` so the " - "signal is not trapped.\n" - f"Offending kill line(s): {default_kill_lines}" - ) - - def test_flag_on_heartbeat_subshell_lifecycle_is_bounded(self, monkeypatch): - """Companion to the trap test: regardless of the trap shape, - the wrapper MUST have an ``EXIT``-time cleanup that stops the - background heartbeat so a clean exit doesn't leave a stray - subshell holding the gateway session open. (The orchestrator's - ``ScriptedProvider`` ban for E2E means we cannot verify this - end-to-end; this is the static guard.) - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The cleanup must be wired into bash's EXIT trap so even an - # unexpected exit path tears the subshell down. Either - # ``trap cleanup EXIT`` or ``trap '' EXIT`` is - # acceptable. - assert "trap " in script and "EXIT" in script, ( - "wrapper must install an EXIT trap to clean up the " - "background heartbeat subshell on any exit path." - ) - - -class TestEventPumpIdleAlertBrcSnapshot: - """Adversarial regression for the v2 idle-alert BRC snapshot bug. - - Plan TASK-2-3 acceptance line 866-867: "alert payload includes - anomaly type, priority, current BRC state". The v2 coder addressed - this by embedding a ``brc_snapshot`` line in the alert detail - sourced from ``${{STATE_JSON:-{}}}``. The bash parameter expansion - is broken: bash's ``${{VAR:-DEFAULT}}`` syntax ends at the FIRST - ``}}`` after ``${{``, so ``${{STATE_JSON:-{}}}`` is parsed as - ``${{STATE_JSON:-{}}`` (default ``{``) followed by a literal - trailing ``}``. When STATE_JSON IS unset the rendered text happens - to read as ``{}}}`` collapsed to a valid empty-object literal by - accident, but when STATE_JSON is populated (the common case during - the event-pump loop) the rendered text appends a STRAY ``}`` to - the JSON document, and ``json.load`` fails with - ``json.decoder.JSONDecodeError: Extra data`` — the snapshot falls - back to ``(unavailable)`` 100% of the time the alert actually has - state to show. - - Verified end-to-end with the rendered bash (slice-2 v2): - - $ STATE_JSON='{"consensus":{"agents":{...}}}' - $ echo "${STATE_JSON:-{}}" \ - | python3 -c 'import sys, json; json.load(sys.stdin)' - json.decoder.JSONDecodeError: Extra data: line 1 column 110 (char 109) - - This is an observability bug, not a correctness wedge: the alert - still fires, but the BRC-state field always reads "(unavailable)" - when state IS available. That defeats the entire point of the - snapshot (tester v1 non-blocker #2). - - Fix: use a temp variable for the default so the bash parser sees - a balanced ``${{...}}``: - - local state_default='{}' - echo "${{STATE_JSON:-$state_default}}" | python3 ... - """ - - def test_flag_on_state_json_default_does_not_corrupt_populated_json(self, monkeypatch): - """The idle-alert BRC snapshot extraction MUST work when - STATE_JSON is populated. Statically check that the rendered - bash does NOT use the broken ``${{STATE_JSON:-{}}}`` pattern. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # The broken pattern. We pin against the exact literal because - # the bash parser fails the same way regardless of variable - # name; if any future code introduces a ``${VAR:-{}}`` it has - # the same bug. - assert "${STATE_JSON:-{}}" not in script, ( - "rendered bash contains `${STATE_JSON:-{}}` which bash " - "parses as `${STATE_JSON:-{}` (default `{`) plus a " - "trailing `}` — populated STATE_JSON values get a stray " - "`}` appended, breaking the downstream `json.load`. The " - "idle-alert BRC-snapshot field will always read " - "`(unavailable)` in the common case. Fix: use a temp var " - "for the default, e.g.\n" - " local state_default='{}'\n" - ' echo "${STATE_JSON:-$state_default}" | python3 ...' - ) - - def test_flag_on_state_json_snapshot_round_trips_populated_json(self, monkeypatch, tmp_path): - """Behavioral round-trip: render the bash, extract the - ``brc_snapshot=$(echo ... | python3 ...)`` block, drive it - with a populated STATE_JSON, and assert the output does NOT - say ``(unavailable)``. - """ - import os - import re - import subprocess - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - cmd = build_consensus_wrapped_command("Prompt") - script = cmd[2] - # Locate the brc_snapshot extraction. The shape evolved across - # cycles: v2 used ``brc_snapshot=$(echo "${STATE_JSON:-{}}" | - # python3 ...)`` (the broken form this test originally pinned); - # v3 uses a separate ``snapshot_input`` variable + explicit - # empty-string check, then ``printf '%s' "$snapshot_input" | - # python3 ...``. We extract from the start of the - # ``raise_idle_alert`` function definition through the - # ``(snapshot unavailable)`` literal so either shape round- - # trips through the harness. - match = re.search( - r"raise_idle_alert\(\) \{(.*?\(snapshot unavailable\)\"\))", - script, - flags=re.DOTALL, - ) - if match is None: - import pytest as _pytest - - _pytest.skip( - "raise_idle_alert / brc_snapshot extraction block not " - "present in rendered bash; behavioral test does not " - "apply." - ) - # The captured group is the function body up through the - # snapshot extraction; trim the leading ``local`` declarations - # so the harness can supply its own STATE_JSON without - # collision. - snapshot_block = match.group(1) - # The captured block lives inside a function body; replace - # ``local`` declarations with plain assignments so the harness - # can run it at the top level of the wrapper script. - snapshot_block = re.sub( - r"^\s*local (\w+)(?: (\w+))?", - lambda m: " ".join(g for g in (m.group(1), m.group(2)) if g), - snapshot_block, - flags=re.MULTILINE, - ) - # Build a minimal harness: define STATE_JSON, run the block, - # echo the result. - harness = ( - 'STATE_JSON=\'{"consensus":{"agents":' - '{"tester":{"confirmed":true,"producer_phase":"WORKING"}},' - '"blocking_agents":["coder"]}}\'\n' - + snapshot_block - + '\necho "RESULT=[$brc_snapshot]"\n' - ) - env = os.environ.copy() - env["EGG_AGENT_ROLE"] = "tester" - result = subprocess.run( - ["bash", "-c", harness], - env=env, - capture_output=True, - text=True, - timeout=5, - ) - # The snapshot MUST contain the role info from the populated - # STATE_JSON, NOT the "(unavailable)" fallback. - assert "RESULT=[" in result.stdout, ( - f"harness did not produce a RESULT line; stdout={result.stdout!r} " - f"stderr={result.stderr!r}" - ) - result_line = next(ln for ln in result.stdout.splitlines() if ln.startswith("RESULT=")) - assert "(unavailable)" not in result_line, ( - "idle-alert BRC snapshot reads `(unavailable)` even when " - "STATE_JSON IS populated — the `${STATE_JSON:-{}}` bash " - "parameter expansion corrupts the JSON with a stray `}` " - "before it reaches `json.load`. The snapshot enhancement " - "ships broken; operators will never see structured state " - "in the alert detail. See test docstring for the fix.\n" - f"Got: {result_line}" - ) - # And the result should contain the role we set. - assert "tester" in result_line, ( - f"snapshot does not contain the EGG_AGENT_ROLE; got {result_line!r}" - ) - - -class TestEventPumpConfirmFailureRaisesIdleAlert: - """(reviewer §1 lock-in) When ``egg-orch consensus confirmed`` - persistently fails on the ``confirm`` arm, the wrapper must NOT - tight-retry. The idle-budget overseer alert is the replacement - for the legacy ``MAX_CONSENSUS_RESTARTS=3`` ceiling -- it MUST - fire when the underlying CLI keeps returning non-zero. - - Pre-fix bug shape (slice-2 v1): the ``confirm)`` arm called - ``note_progress`` unconditionally after the CLI returned, which - reset ``LAST_PROGRESS`` and both ``ALERTED_AT_*`` latches every - iteration. ``check_idle_budget`` therefore never observed a - growing idle and the alert never fired -- a tight retry loop - with zero operator-visible signal. - - Post-fix: ``note_progress`` only fires on rc==0. A persistent - non-zero rc lets the idle counter accrue; ``check_idle_budget`` - fires the OVERSEER_ALERT at the configured budget. - """ - - def test_persistent_confirm_failure_fires_overseer_alert(self, tmp_path, monkeypatch): - """End-to-end behavioural test of the §1 + §6.2 lock-in. - - Drive the rendered event-pump bash against stubbed - ``egg-orch`` / ``python3`` shims: - - ``brc get-state`` → role unconfirmed - - ``brc next-action`` → ``{"action":"confirm"}`` - - ``consensus confirmed`` → exit 1 every call (persistent - failure) - - ``overseer alert`` → record the call to a log file - - everything else → noop / exit 0 - - With ``EGG_BRC_IDLE_BUDGET_MIN=0`` the very first - ``check_idle_budget`` after the failing confirm trips the - ``idle >= 2*budget`` (= 0) branch -- exactly once, post-fix. - - Discrimination: per-iteration count of the alert is the - differential between the bug and the fix. - - - Pre-fix (note_progress reset on every iteration AND - ALERTED_AT_DOUBLE reset every iteration -- the combined - §1 + §6.2 bug): every loop iter resets the latch so - ``check_idle_budget`` re-fires the 2x alert *every* - iteration. Within the 8s timeout below, observed count - is several. - - Post-fix (rc-gated note_progress AND sticky - ALERTED_AT_DOUBLE): alert fires once on the first - iteration; subsequent iterations see the sticky latch - and do not re-fire. - - The ``count == 1`` assertion locks in the worst-case - regression where BOTH §1 and §6.2 regress together: the - action arm calls ``note_progress`` on every iteration AND - ``note_progress`` resets ``ALERTED_AT_DOUBLE``, so the - 2x-budget alert re-fires every loop iteration, yielding - ``count >> 1``. The fix yields exactly 1. - - Note: with ``EGG_BRC_IDLE_BUDGET_MIN=0`` either regression - in isolation still yields ``count == 1`` (a §1-only - regression rearms ``LAST_PROGRESS`` but ``ALERTED_AT_DOUBLE`` - stays sticky after iter-1; a §6.2-only regression never - reaches the reset path because rc-gated ``note_progress`` - is never called on persistent failure). The combined - regression is the alert-flood scenario worth catching here. - """ - # ``EGG_BRC_EVENT_PUMP`` is silently inert after slice-4 task-4-2 - # (the env-flag read was deleted along with the legacy template); - # this ``setenv`` is harmlessly retained so a future regression - # that re-introduces a flag-gated branch trips the test if it - # depends on the env. ``build_consensus_wrapped_command`` always - # emits the event-pump template body now. - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - # Stub directory on PATH ahead of the real egg-orch. - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - confirm_log = tmp_path / "confirm_calls.log" - alert_log = tmp_path / "alert_calls.log" - general_log = tmp_path / "egg_orch.log" - - # Mock egg-orch: route on the first two positional words so we - # can recognise ``brc get-state`` / ``brc next-action`` / - # ``consensus confirmed`` / ``overseer alert`` etc. - mock_orch = bin_dir / "egg-orch" - mock_orch.write_text( - "#!/bin/bash\n" - f'echo "$@" >> {shlex.quote(str(general_log))}\n' - 'sub="$1 $2"\n' - 'case "$sub" in\n' - ' "brc get-state")\n' - ' echo \'{"consensus":{"agents":{"coder":{"confirmed":false,' - '"producer_phase":"WAITING_FOR_REVIEW"}},"is_complete":false}}\'\n' - " ;;\n" - ' "brc next-action")\n' - ' echo \'{"action":"confirm"}\'\n' - " ;;\n" - ' "consensus confirmed")\n' - f" echo confirm_call >> {shlex.quote(str(confirm_log))}\n" - " exit 1\n" - " ;;\n" - ' "overseer alert")\n' - f' echo "alert: $*" >> {shlex.quote(str(alert_log))}\n' - " ;;\n" - " *)\n" - " # message heartbeat, message wait-loop, etc -- benign no-ops.\n" - " ;;\n" - "esac\n" - "exit 0\n" - ) - os.chmod(str(mock_orch), 0o755) # nosec B103 - - # Stub python3 so the agent invocation arm (not exercised here - # because next-action == ``confirm``) doesn't accidentally run - # the real Agent SDK if a future regression flips the action. - # Forward inline ``python3 -c`` invocations (which the wrapper - # uses for JSON parsing) to the real interpreter. - real_python = sys.executable - mock_python = bin_dir / "python3" - mock_python.write_text( - "#!/bin/bash\n" - 'if [ "$1" = "-c" ] || [ "$1" = "-" ]; then\n' - f' exec {shlex.quote(real_python)} "$@"\n' - "fi\n" - "# Agent SDK invocation path -- treat as success no-op.\n" - "exit 0\n" - ) - os.chmod(str(mock_python), 0o755) # nosec B103 - - # Build the wrapper with the flag on; idle budget of 0 makes - # ``check_idle_budget`` fire on the first non-progress - # iteration. - env = os.environ.copy() - env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" - env["EGG_BRC_EVENT_PUMP"] = "true" - env["EGG_BRC_IDLE_BUDGET_MIN"] = "0" - env["EGG_AGENT_ROLE"] = "coder" - env["EGG_PIPELINE_ID"] = "test-pipeline" - env["EGG_CONCURRENT_MODE"] = "true" - - cmd = build_consensus_wrapped_command("Prompt") - # The wrapper loops forever; bound the test with a short - # timeout. By that time, several confirm attempts and at - # least one overseer alert must have been recorded if the - # §1 fix is in place. - try: - subprocess.run( - cmd, - env=env, - capture_output=True, - text=True, - timeout=8, - ) - except subprocess.TimeoutExpired: - pass # expected — the wrapper loops; we bound it. - - assert confirm_log.exists(), ( - "wrapper did not call `egg-orch consensus confirmed` at " - "all on the confirm arm -- the event-pump loop may not " - "have reached the confirm action. egg-orch log:\n" - + (general_log.read_text() if general_log.exists() else "(empty)") - ) - confirm_attempts = confirm_log.read_text().count("confirm_call") - assert confirm_attempts >= 1, f"expected >= 1 confirm attempts, got {confirm_attempts}" - - assert alert_log.exists(), ( - "§1 regression: `egg-orch consensus confirmed` failed " - f"{confirm_attempts} times but the overseer idle-budget " - "alert never fired. The pre-fix `note_progress` reset " - "ran unconditionally on every confirm-arm iteration, " - "resetting LAST_PROGRESS and the ALERTED_AT_* latches so " - "`check_idle_budget` never observed a growing idle. " - "Post-fix: `note_progress` only fires on rc==0; " - "persistent confirm failure must surface as an " - "OVERSEER_ALERT. egg-orch log:\n" + general_log.read_text() - ) - alert_text = alert_log.read_text() - assert "stuck-phase-transition" in alert_text, ( - f"overseer alert fired but with the wrong anomaly type. Got:\n{alert_text}" - ) - # Lock-in for the combined §1 + §6.2 regression (reviewer - # follow-up on PR #2926, commit 022fad4): the bug fires the - # 2x-budget alert *every* loop iteration because (a) the - # action arm called ``note_progress`` regardless of rc and - # (b) ``note_progress`` itself reset ``ALERTED_AT_DOUBLE``. - # The fix gates ``note_progress`` on rc==0 AND keeps the 2x - # latch sticky for the loop lifetime -> exactly one alert. - alert_count = alert_text.count("stuck-phase-transition") - assert alert_count == 1, ( - "§1 + §6.2 regression: persistent confirm failure produced " - f"{alert_count} overseer alerts, expected exactly 1. " - "Pre-fix, the action arm reset both LAST_PROGRESS and the " - "ALERTED_AT_DOUBLE latch on every iteration, so " - "`check_idle_budget` re-fired the 2x-budget alert each " - "loop. Post-fix, `note_progress` is gated on rc==0 (so " - "persistent failure does not reset state) AND " - "`ALERTED_AT_DOUBLE` is sticky (so transient progress " - "later in the loop's life cannot re-arm the page). " - f"Alert text:\n{alert_text}" - ) - - -class TestEventPumpAgentFailStreakEscalation: - """(#3138) The ``propose|ack|nack`` arm must not retry a - deterministic fast failure indefinitely at a flat 1 s cadence. - - Pre-fix bug shape (first issue-3077 run, 2026-06-11): the refiner's - agent invocation failed pre-SDK-init (unknown model alias, rc=1 in - <1 s) and the arm retried it 160+ consecutive times, one every - ~2-3 s, with no backoff growth and no escalation. ``AGENT_FAIL_STREAK`` - was computed and logged but nothing acted on it; the only detection - was the overseer's generic heartbeat-silence anomaly ~8 minutes in. - - Post-fix, the arm has parity with its two siblings: - - backoff parity with the ``confirm`` arm: linear in the streak - (streak x 2 s), capped at 30 s; - - escalation parity beyond the ``fetch_next_action`` latches: a - sticky log warning at streak 5 and a sticky OVERSEER_ALERT - (anomaly ``agent-invocation-fail-streak``) at streak 10 whose - detail classifies sub-second failures as configuration-class. - The wrapper still never self-FAILs (post-#2908 design). - """ - - def test_template_has_agent_arm_backoff_and_streak_latches(self, monkeypatch): - """Render-level lock-in: the agent arm computes a capped linear - backoff from ``AGENT_FAIL_STREAK``, defines both sticky latches, - raises the streak alert, and no longer contains the flat - ``sleep 1`` retry floor as executable bash.""" - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - script = build_consensus_wrapped_command("Prompt")[-1] - - assert "agent_backoff_secs=$(( AGENT_FAIL_STREAK * 2 ))" in script, ( - "agent arm must grow its retry sleep linearly with the " - "fail streak (parity with the confirm arm)" - ) - assert 'sleep "$agent_backoff_secs"' in script - assert "AGENT_FAIL_ALERTED_5=false" in script - assert "AGENT_FAIL_ALERTED_10=false" in script - assert "raise_agent_fail_alert" in script - assert "agent-invocation-fail-streak" in script, ( - "streak escalation must reach the overseer as a distinctly " - "named anomaly, not only a cw_log line" - ) - # The pre-#3138 flat retry floor must not survive as executable - # bash (a backtick-quoted mention in a comment is fine). - executable_lines = [ - line.strip() for line in script.splitlines() if not line.lstrip().startswith("#") - ] - assert "sleep 1" not in executable_lines, ( - "#3138 regression: the agent arm's flat `sleep 1` retry " - "floor is back — a pre-SDK-init failure would again retry " - "indefinitely at the floor" - ) - - def test_persistent_agent_failure_backs_off_and_fires_streak_alert(self, tmp_path, monkeypatch): - """End-to-end behavioural test of the #3138 fix. - - Drive the rendered event-pump bash against stubbed shims: - - ``brc get-state`` → role unconfirmed, consensus incomplete - - ``brc next-action`` → ``{"action":"propose"}`` every call - - ``python3 -m egg_agent ...`` → exit 1 immediately (the - pre-SDK-init fast-fail signature from #3136; inline - ``python3 -c`` JSON parsing forwards to the real - interpreter) - - ``overseer alert`` → record the call to a log file - - ``sleep`` → record the requested duration, then sleep a - scaled-down 0.05 s so the loop reaches a deep streak - within the test timeout without real waiting - - Assertions: - - the recorded sleep durations are exactly the expected - capped-linear sequence (2, 4, 6, ... capped at 30) — i.e. - backoff grows with the streak and respects the cap; - - the streak reached at least 10 (so the escalation path ran); - - the ``agent-invocation-fail-streak`` overseer alert fired - EXACTLY once (sticky latch — pre-fix the signal never - fired; an unsticky latch would flood); - - the alert detail carries the sub-second - configuration-class classification. - """ - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") - bin_dir = tmp_path / "bin" - bin_dir.mkdir() - alert_log = tmp_path / "alert_calls.log" - sleep_log = tmp_path / "sleep_calls.log" - general_log = tmp_path / "egg_orch.log" - - mock_orch = bin_dir / "egg-orch" - mock_orch.write_text( - "#!/bin/bash\n" - f'echo "$@" >> {shlex.quote(str(general_log))}\n' - 'sub="$1 $2"\n' - 'case "$sub" in\n' - ' "brc get-state")\n' - ' echo \'{"consensus":{"agents":{"coder":{"confirmed":false,' - '"producer_phase":"WAITING_FOR_REVIEW"}},"is_complete":false}}\'\n' - " ;;\n" - ' "brc next-action")\n' - ' echo \'{"action":"propose"}\'\n' - " ;;\n" - ' "overseer alert")\n' - f' echo "alert: $*" >> {shlex.quote(str(alert_log))}\n' - " ;;\n" - " *)\n" - " ;;\n" - "esac\n" - "exit 0\n" - ) - os.chmod(str(mock_orch), 0o755) # nosec B103 - - # Inline ``python3 -c`` / stdin parsing forwards to the real - # interpreter; the ``python3 -m egg_agent`` invocation path - # fast-fails like an unknown-model session-init crash (#3136). - real_python = sys.executable - mock_python = bin_dir / "python3" - mock_python.write_text( - "#!/bin/bash\n" - 'if [ "$1" = "-c" ] || [ "$1" = "-" ]; then\n' - f' exec {shlex.quote(real_python)} "$@"\n' - "fi\n" - "exit 1\n" - ) - os.chmod(str(mock_python), 0o755) # nosec B103 - - # Record requested sleep durations, then sleep a scaled-down - # constant so backoff growth is observable without real waiting. - # ``command -p`` resolves the real sleep from the default PATH - # (this stub shadows ``sleep`` on the test PATH, so a bare - # ``sleep`` here would recurse). - mock_sleep = bin_dir / "sleep" - mock_sleep.write_text( - "#!/bin/bash\n" - f'echo "$1" >> {shlex.quote(str(sleep_log))}\n' - "command -p sleep 0.05 2>/dev/null || exit 0\n" - "exit 0\n" - ) - os.chmod(str(mock_sleep), 0o755) # nosec B103 - - env = os.environ.copy() - env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" - env["EGG_BRC_EVENT_PUMP"] = "true" - # Leave the idle budget at its 30-min default: the streak alert - # must come from the agent arm, not the idle-budget safety net. - env.pop("EGG_BRC_IDLE_BUDGET_MIN", None) - env["EGG_AGENT_ROLE"] = "coder" - env["EGG_PIPELINE_ID"] = "test-pipeline" - env["EGG_CONCURRENT_MODE"] = "true" - - cmd = build_consensus_wrapped_command("Prompt") - try: - subprocess.run( - cmd, - env=env, - capture_output=True, - text=True, - # Needs >=10 loop iterations, each spawning several - # egg-orch/python3 subprocesses, to exercise the streak-10 - # escalation. 20s gives headroom on a loaded CI host so the - # ``len(observed) >= 10`` assertion below doesn't flake. - timeout=20, - ) - except subprocess.TimeoutExpired: - pass # expected — the wrapper loops; we bound it. - - assert sleep_log.exists(), ( - "agent arm never slept after a failed invocation -- the " - "event-pump loop may not have reached the propose arm. " - "egg-orch log:\n" + (general_log.read_text() if general_log.exists() else "(empty)") - ) - observed = [int(line) for line in sleep_log.read_text().split() if line.isdigit()] - expected = [min(2 * (i + 1), 30) for i in range(len(observed))] - assert observed == expected, ( - "#3138 regression: agent-arm retry sleeps must follow the " - "capped-linear backoff (streak x 2 s, capped 30 s, parity " - f"with the confirm arm). Observed {observed[:20]}, expected " - f"{expected[:20]}" - ) - assert len(observed) >= 10, ( - "test did not reach streak 10 within the timeout; the " - f"escalation path was not exercised (streak={len(observed)})" - ) - - assert alert_log.exists(), ( - "#3138 regression: 10+ consecutive fast agent-invocation " - "failures did not raise an overseer alert from the wrapper. " - "Pre-fix, AGENT_FAIL_STREAK was computed and discarded; the " - "operator only heard about the loop ~8 minutes later via " - "the overseer's generic heartbeat-silence anomaly. egg-orch " - "log:\n" + general_log.read_text() - ) - alert_text = alert_log.read_text() - alert_count = alert_text.count("agent-invocation-fail-streak") - assert alert_count == 1, ( - "streak alert must fire exactly once per wrapper lifetime " - f"(sticky AGENT_FAIL_ALERTED_10 latch); got {alert_count}. " - f"Alert text:\n{alert_text}" - ) - assert "configuration-class" in alert_text, ( - "sub-second failures must be classified as configuration-" - "class (unknown model alias, auth, prompt-rendering crash) " - f"in the alert detail. Alert text:\n{alert_text}" - ) - class TestEventPumpInvokesComposer: """Pin the wrapper template's ``invoke_agent_for_event`` invocation @@ -1398,22 +46,15 @@ class TestEventPumpInvokesComposer: ``python3 "$script_path"`` call shape. """ - def test_flag_on_template_defines_invoke_agent_for_event(self, monkeypatch) -> None: - from consensus_wrapper import build_consensus_wrapped_command - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") + def test_template_defines_invoke_agent_for_event(self) -> None: cmd = build_consensus_wrapped_command("hello") script = cmd[2] assert "invoke_agent_for_event()" in script, ( "Wrapper template must define `invoke_agent_for_event` so the " - "per-event prompt composition is in place for slice-3 / " - "slice-4 wiring." + "per-event prompt composition is in place." ) - def test_flag_on_template_references_event_prompt_script(self, monkeypatch) -> None: - from consensus_wrapper import build_consensus_wrapped_command - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") + def test_template_references_event_prompt_script(self) -> None: cmd = build_consensus_wrapped_command("hello") script = cmd[2] # Either the hard-coded production path OR the override env var @@ -1428,10 +69,7 @@ def test_flag_on_template_references_event_prompt_script(self, monkeypatch) -> N # The actual script-path env var name is the documented seam. assert "EGG_EVENT_PROMPT_SCRIPT" in script - def test_flag_on_template_re_exports_memory_env_var(self, monkeypatch) -> None: - from consensus_wrapper import build_consensus_wrapped_command - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") + def test_template_re_exports_memory_env_var(self) -> None: cmd = build_consensus_wrapped_command("hello") script = cmd[2] # The re-export line must be present so the env-var contract @@ -1439,16 +77,13 @@ def test_flag_on_template_re_exports_memory_env_var(self, monkeypatch) -> None: # env vars (``EGG_AGENT_ROLE`` / ``EGG_BASE_BRANCH`` / # ``EGG_REPO_PATH`` / ``EGG_BRC_MEMORY``) must be re-exported # on the python3 invocation per the reviewer_holistic v2 - # follow-up — the earlier shape relied on ``EGG_REPO_PATH`` - # being exported by the parent shell, which works in - # production today but breaks symmetry with the in-source - # comment that lists all four as wrapper-supplied. + # follow-up. assert "EGG_BRC_MEMORY=" in script assert "EGG_AGENT_ROLE=" in script assert "EGG_BASE_BRANCH=" in script assert "EGG_REPO_PATH=" in script - def test_flag_on_template_env_prefix_attaches_to_python3_not_printf(self, monkeypatch) -> None: + def test_template_env_prefix_attaches_to_python3_not_printf(self) -> None: """The env-var prefix must attach to ``python3`` (RHS of the pipe), not ``printf`` (LHS). The earlier form attached only to ``printf`` and ``python3`` inherited from the parent shell — a @@ -1457,9 +92,6 @@ def test_flag_on_template_env_prefix_attaches_to_python3_not_printf(self, monkey break if the parent shell didn't (reviewer_contract NACK v1 finding #1 + reviewer_code NACK v1 finding #1). """ - from consensus_wrapper import build_consensus_wrapped_command - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") cmd = build_consensus_wrapped_command("hello") script = cmd[2] # The pipe must run printf first (LHS) without env-var prefix, @@ -1479,12 +111,7 @@ def test_flag_on_template_env_prefix_attaches_to_python3_not_printf(self, monkey "(RHS of the pipe) per the v1 NACK fix." ) - def test_flag_on_template_invokes_python3_with_script_path_and_action( - self, monkeypatch - ) -> None: - from consensus_wrapper import build_consensus_wrapped_command - - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") + def test_template_invokes_python3_with_script_path_and_action(self) -> None: cmd = build_consensus_wrapped_command("hello") script = cmd[2] # The call shape: ``python3 "$script_path" "$action"``. @@ -1494,13 +121,6 @@ def test_flag_on_template_invokes_python3_with_script_path_and_action( "action argv would silently break the CLI contract." ) - # Note: slice-2's ``test_flag_off_legacy_template_does_not_reference_event_prompt`` - # was deleted by slice-4 task-4-2. The legacy template is gone; the - # event-pump template now unconditionally references ``event_prompt.py`` - # and ``invoke_agent_for_event``. See - # ``test_invokes_event_prompt_composer_script`` (above) for the - # post-deletion positive invariant. - class TestEffortFlag: """``effort`` threads into the agent command prefix as ``--effort``. @@ -1533,54 +153,56 @@ class TestSyncToProposals: worktree. The wrapper now performs that sync deterministically: before an ``ack``/``nack`` invocation it merges each pending producer's ``proposal_commit_sha`` into the reviewer worktree, - fail-soft at every step. + fail-soft at every step. Under the one-shot handler (#3164) the + review-action dispatch is gated on ``ONE_SHOT_ACTION``. """ - def _script(self, monkeypatch) -> str: - monkeypatch.setenv("EGG_BRC_EVENT_PUMP", "true") + def _script(self) -> str: return build_consensus_wrapped_command("Prompt")[2] - def test_template_defines_sync_to_proposals(self, monkeypatch): - script = self._script(monkeypatch) + def test_template_defines_sync_to_proposals(self): + script = self._script() assert "sync_to_proposals() {" in script - def test_sync_runs_only_for_review_actions(self, monkeypatch): + def test_sync_runs_only_for_review_actions(self): """The sync call must be gated on ack/nack — a producer's own ``propose`` invocation must NOT merge peers' commits into its worktree (R11a: propose own work first, peer state irrelevant). + Under the one-shot handler (#3164) the gate reads + ``ONE_SHOT_ACTION``. """ - script = self._script(monkeypatch) - guard = 'if [ "$ACTION" = "ack" ] || [ "$ACTION" = "nack" ]; then' + script = self._script() + guard = 'if [ "$ONE_SHOT_ACTION" = "ack" ] || [ "$ONE_SHOT_ACTION" = "nack" ]; then' assert guard in script # The call rides inside that guard, with the event payload. guarded_block = script.split(guard, 1)[1].split("fi", 1)[0] - assert 'sync_to_proposals "$EVENT_PAYLOAD"' in guarded_block + assert 'sync_to_proposals "$ONE_SHOT_PAYLOAD"' in guarded_block - def test_sync_precedes_agent_invocation(self, monkeypatch): + def test_sync_precedes_agent_invocation(self): """Ordering invariant: the worktree must be synced BEFORE the one-shot agent runs, or the tester still reviews a stale tree. """ - script = self._script(monkeypatch) - sync_pos = script.index('sync_to_proposals "$EVENT_PAYLOAD"') - invoke_pos = script.index('invoke_agent_for_event "$ACTION" "$EVENT_PAYLOAD"') + script = self._script() + sync_pos = script.index('sync_to_proposals "$ONE_SHOT_PAYLOAD"') + invoke_pos = script.index('invoke_agent_for_event "$ONE_SHOT_ACTION" "$ONE_SHOT_PAYLOAD"') assert sync_pos < invoke_pos - def test_sha_extraction_is_hex_validated(self, monkeypatch): + def test_sha_extraction_is_hex_validated(self): """The producer-supplied SHA is interpolated into git argv; the extractor must hex-validate (7-64 chars) so shell metacharacters and non-hex sentinels (RECONSTRUCTED_NO_SHA) never reach git. """ - script = self._script(monkeypatch) + script = self._script() assert "[0-9a-fA-F]{7,64}" in script assert "fullmatch" in script - def test_merge_failure_is_fail_soft(self, monkeypatch): + def test_merge_failure_is_fail_soft(self): """A conflicting merge must abort and continue — the per-event prompt's ``git show`` reads (#3078) remain the fallback; the agent invocation must never be blocked on the sync. """ - script = self._script(monkeypatch) + script = self._script() assert "merge --abort" in script # The function never propagates failure into the action arm. fn_body = script.split("sync_to_proposals() {", 1)[1] @@ -1612,7 +234,7 @@ def _extract_sync_harness(self, script: str, repo: str, payload: str) -> str: + '\necho "SYNC_RC=$?"\n' ) - def test_behavioral_merge_and_metachar_filter(self, tmp_path, monkeypatch): + def test_behavioral_merge_and_metachar_filter(self, tmp_path): """End-to-end: a real proposal SHA on a producer branch is merged into the reviewer's checkout (the proposed artifact becomes Read-able); a shell-metachar SHA is filtered before @@ -1662,7 +284,7 @@ def git(*args): ] } ) - script = self._script(monkeypatch) + script = self._script() harness = self._extract_sync_harness(script, str(repo), payload) result = subprocess.run( ["bash", "-c", harness], @@ -1685,7 +307,7 @@ def git(*args): # The metachar SHA was filtered, not executed/attempted. assert "rm -rf" not in result.stderr - def test_behavioral_unresolvable_sha_logs_and_continues(self, tmp_path, monkeypatch): + def test_behavioral_unresolvable_sha_logs_and_continues(self, tmp_path): """A well-formed but unknown SHA logs the git-show fallback and exits 0 — never fails the action arm.""" import json as _json @@ -1696,7 +318,7 @@ def test_behavioral_unresolvable_sha_logs_and_continues(self, tmp_path, monkeypa payload = _json.dumps( {"pending_reviews": [{"producer": "x", "proposal_commit_sha": "a" * 40}]} ) - script = self._script(monkeypatch) + script = self._script() harness = self._extract_sync_harness(script, str(repo), payload) result = subprocess.run(["bash", "-c", harness], capture_output=True, text=True, timeout=30) assert "SYNC_RC=0" in result.stdout @@ -1736,16 +358,10 @@ class TestSyncOutcomesAndBanner: # covered at the wrapper level." _OUTCOMES = ("merged", "already-ancestor", "unresolvable", "merge-failed") - def _script(self, monkeypatch) -> str: - # ``EGG_BRC_EVENT_PUMP`` was deleted by slice-4 task-4-2 and is - # silently inert (see ``TestEventPumpTemplateSelection``); the - # event-pump is the only template. ``monkeypatch`` is kept on - # the signature so a future template-selection toggle can be - # added back without changing every call site. - del monkeypatch + def _script(self) -> str: return build_consensus_wrapped_command("Prompt")[2] - def test_template_records_all_four_outcomes(self, monkeypatch): + def test_template_records_all_four_outcomes(self): """Each of the four per-SHA outcome tokens (``merged``, ``already-ancestor``, ``unresolvable``, ``merge-failed``) is present in the wrapper template. A future refactor that drops @@ -1753,7 +369,7 @@ def test_template_records_all_four_outcomes(self, monkeypatch): ``merged`` log line — would re-orphan a path the banner needs to distinguish. """ - script = self._script(monkeypatch) + script = self._script() missing = [o for o in self._OUTCOMES if o not in script] assert not missing, ( "Slice-1 outcome contract requires all four per-SHA outcome " @@ -1762,7 +378,7 @@ def test_template_records_all_four_outcomes(self, monkeypatch): "merged, already-ancestor, unresolvable, merge-failed." ) - def test_template_emits_not_synced_banner_text(self, monkeypatch): + def test_template_emits_not_synced_banner_text(self): """The banner string the agent sees on a failure outcome is pinned to the architect's wording ("worktree NOT synced ...") AND wired into the ``SYNC_FAILURE_BANNERS`` accumulator. The @@ -1770,7 +386,7 @@ def test_template_emits_not_synced_banner_text(self, monkeypatch): downgrades the banner to a dead comment; pinning the ``SYNC_FAILURE_BANNERS+=`` append site catches that too. """ - script = self._script(monkeypatch) + script = self._script() assert "NOT synced" in script, ( "Wrapper template must emit the 'worktree NOT synced' " "banner so a reviewer whose sync silently failed cannot " @@ -1810,25 +426,25 @@ def test_template_emits_not_synced_banner_text(self, monkeypatch): f"{append_sites}" ) - def test_template_references_git_show_fallback_in_banner(self, monkeypatch): + def test_template_references_git_show_fallback_in_banner(self): """The banner steers reviewers at the rendered ``git show`` delta commands (#3078 served reads) — that is the live replacement channel R1 is non-silently surfacing. The substring ``git show`` must appear inside the banner-bearing region so the agent has a next step beyond "sync failed". """ - script = self._script(monkeypatch) + script = self._script() # The banner-bearing region runs from the start of - # ``invoke_agent_for_event`` (where ``$prompt`` is composed) to - # the start of ``sync_to_proposals`` (whose closing fence ends - # before the main loop). The exact placement of the prepend - # logic — sync function vs invoke function vs a small helper — - # is the coder's call; we only require both regions, taken - # together, to carry the banner-and-fallback text. + # ``invoke_agent_for_event`` / ``sync_to_proposals`` (where the + # banner is produced and consumed) to the one-shot event handler + # section that dispatches the actions. The exact placement of the + # prepend logic — sync function vs invoke function vs a small + # helper — is the coder's call; we only require both regions, + # taken together, to carry the banner-and-fallback text. invoke_start = script.index("invoke_agent_for_event() {") sync_start = script.index("sync_to_proposals() {") body_lo = min(invoke_start, sync_start) - body_hi = script.index("# --- main event-pump loop ---") + body_hi = script.index("# --- one-shot event handler (#3164)") body = script[body_lo:body_hi] assert "NOT synced" in body, ( "Banner text must live inside the sync/invoke region — " @@ -1953,8 +569,8 @@ def git(*args): git("commit", "-qm", "base") return repo, git - def _run_harness(self, tmp_path, monkeypatch, payload, capture, stub): - script = self._script(monkeypatch) + def _run_harness(self, tmp_path, payload, capture, stub): + script = self._script() harness = self._build_harness(script, str(tmp_path / "repo"), payload, str(capture), stub) result = subprocess.run( ["bash", "-c", harness, "harness", payload], @@ -1970,9 +586,8 @@ def _run_harness(self, tmp_path, monkeypatch, payload, capture, stub): }, ) # Sync function and invoke function must each exit zero; the - # idle-budget safety net keeps the event-pump moving even on a - # composer error, and the slice-1 banner mechanism rides on that - # invariant (banner is a prompt prefix, not a hard failure). + # slice-1 banner mechanism rides on that invariant (banner is a + # prompt prefix, not a hard failure). assert "HARNESS_RC=0" in result.stdout, ( "Harness must exit 0; the slice-1 banner is a prompt " "prefix, not a hard failure. " @@ -1980,7 +595,7 @@ def _run_harness(self, tmp_path, monkeypatch, payload, capture, stub): ) return result - def test_behavioral_merged_outcome_no_banner(self, tmp_path, monkeypatch): + def test_behavioral_merged_outcome_no_banner(self, tmp_path): """Successful merge → ``merged`` outcome recorded, agent-visible prompt is byte-identical to the composed prompt (no banner). @@ -2009,7 +624,7 @@ def test_behavioral_merged_outcome_no_banner(self, tmp_path, monkeypatch): ) capture = tmp_path / "agent_prompt.txt" stub = self._stub_composer(tmp_path) - result = self._run_harness(tmp_path, monkeypatch, payload, capture, stub) + result = self._run_harness(tmp_path, payload, capture, stub) assert "merged" in result.stderr, ( f"``merged`` outcome must be observable at the wrapper level. stderr={result.stderr!r}" @@ -2021,7 +636,7 @@ def test_behavioral_merged_outcome_no_banner(self, tmp_path, monkeypatch): f"prefix. Got: {prompt!r}" ) - def test_behavioral_already_ancestor_outcome_no_banner(self, tmp_path, monkeypatch): + def test_behavioral_already_ancestor_outcome_no_banner(self, tmp_path): """SHA already in HEAD ancestry → ``already-ancestor`` outcome, agent-visible prompt unchanged.""" import json as _json @@ -2039,7 +654,7 @@ def test_behavioral_already_ancestor_outcome_no_banner(self, tmp_path, monkeypat ) capture = tmp_path / "agent_prompt.txt" stub = self._stub_composer(tmp_path) - result = self._run_harness(tmp_path, monkeypatch, payload, capture, stub) + result = self._run_harness(tmp_path, payload, capture, stub) assert "already-ancestor" in result.stderr, ( "``already-ancestor`` outcome must be observable at the " @@ -2051,7 +666,7 @@ def test_behavioral_already_ancestor_outcome_no_banner(self, tmp_path, monkeypat f"byte-identical. Got: {prompt!r}" ) - def test_behavioral_unresolvable_outcome_emits_banner(self, tmp_path, monkeypatch): + def test_behavioral_unresolvable_outcome_emits_banner(self, tmp_path): """Well-formed but unknown SHA → ``unresolvable`` outcome AND the "NOT synced" banner reaches the agent prompt. @@ -2068,7 +683,7 @@ def test_behavioral_unresolvable_outcome_emits_banner(self, tmp_path, monkeypatc ) capture = tmp_path / "agent_prompt.txt" stub = self._stub_composer(tmp_path) - result = self._run_harness(tmp_path, monkeypatch, payload, capture, stub) + result = self._run_harness(tmp_path, payload, capture, stub) assert "unresolvable" in result.stderr, ( "``unresolvable`` outcome must be observable at the " @@ -2097,7 +712,7 @@ def test_behavioral_unresolvable_outcome_emits_banner(self, tmp_path, monkeypatc f"substitution. Prompt: {prompt!r}" ) - def test_behavioral_merge_failed_outcome_emits_banner(self, tmp_path, monkeypatch): + def test_behavioral_merge_failed_outcome_emits_banner(self, tmp_path): """Conflicting merge → ``merge-failed`` outcome AND the "NOT synced" banner reaches the agent prompt. @@ -2133,7 +748,7 @@ def test_behavioral_merge_failed_outcome_emits_banner(self, tmp_path, monkeypatc ) capture = tmp_path / "agent_prompt.txt" stub = self._stub_composer(tmp_path) - result = self._run_harness(tmp_path, monkeypatch, payload, capture, stub) + result = self._run_harness(tmp_path, payload, capture, stub) assert "merge-failed" in result.stderr, ( "``merge-failed`` outcome must be observable at the " @@ -2157,275 +772,180 @@ def test_behavioral_merge_failed_outcome_emits_banner(self, tmp_path, monkeypatc # =========================================================================== -# Slice-1 (#3064 task-1-2): ownership flag + one-shot wrapper arm. -# -# These tests are TEST-FIRST: they pin the slice-1 contract the coder's -# parallel implementation (task-1-1) must satisfy. The pod-default golden -# snapshot passes against the pre-#3064 rendering today (R1 guard); the -# orchestrator-mode one-shot arm and the ``env_config`` accessor tests go -# green once the coder's slice-1 implementation is integrated. -# -# Contract pinned here (the plan, "Slice 1 — Ownership flag + one-shot -# wrapper arm"): -# * ``env_config.get_event_loop_owner()`` reads ``EGG_EVENT_LOOP_OWNER`` -# ∈ {``pod`` (default), ``orchestrator``}; an unrecognised value is -# rejected loudly by raising ``ValueError`` (the deliberate exception -# to the module's usual never-raise/warn-and-default convention — -# #3023: a wrong ownership mode deadlocks BRC, so a typo must surface -# at read time rather than masquerade as the ``pod`` default). -# * ``build_consensus_wrapped_command`` selects the rendering by owner: -# ``pod``/unset → today's in-pod event-pump template (byte-identical, -# golden-pinned); ``orchestrator`` → the single-event one-shot arm. -# * The one-shot arm skips the blocking wait-loop and background -# heartbeat, re-checks ``brc next-action`` once (stale ⇒ exit 0, no -# agent invocation — the dedupe backstop), invokes the agent exactly -# once on a fresh ``propose|ack|nack`` event, and exits with the -# agent's (#2908-classified) exit code. ``confirm``/``complete`` never -# reach the one-shot arm — an injected terminal action is rejected -# loudly. +# One-shot wrapper (#3164): the in-pod BRC event-loop wait arm was retired. +# ``build_consensus_wrapped_command`` now renders a single rendering — a +# one-shot event handler driven by the injected ``EGG_EVENT_ACTION`` — with +# no blocking wait-loop, no background heartbeat, and no idle-budget / +# fail-streak machinery (those re-homed to the orchestrator event loop). +# The rendering is IDENTICAL regardless of any env var: there is no +# ownership flag and no ``EGG_EVENT_LOOP_OWNER`` mode. # =========================================================================== _GOLDEN_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "golden") -_POD_DEFAULT_GOLDEN_PATH = os.path.join(_GOLDEN_DIR, "event_pump_wrapper_pod_default.sh.golden") +_GOLDEN_PATH = os.path.join(_GOLDEN_DIR, "event_pump_wrapper.sh.golden") -def _read_pod_default_golden() -> str: - with open(_POD_DEFAULT_GOLDEN_PATH, encoding="utf-8") as fh: +def _read_golden() -> str: + with open(_GOLDEN_PATH, encoding="utf-8") as fh: return fh.read() -class TestPodDefaultWrapperGoldenSnapshot: - """R1 guard (#3064 slice-1 acceptance: "Golden-file test fails on ANY - drift of the pod-default wrapper rendering"). +class TestWrapperGoldenSnapshot: + """Byte-for-byte guard on the one-shot wrapper rendering (#3164). - The #3023 post-mortem deadlocked BRC by silencing the in-pod loop with - nothing replacing it. Slice-1 lands the ownership guard *dormant*: with - ``EGG_EVENT_LOOP_OWNER`` unset or ``pod`` the generated wrapper MUST be - byte-identical to the pre-#3064 event-pump rendering, so the guard can - never regress the production (pod) path silently. + A golden-file snapshot fails on ANY drift of the rendered wrapper so + a change to the one-shot path can never regress the production path + silently. - The fixture deliberately carries the ``.sh.golden`` suffix (not ``.sh``) - so ``make lint-shell`` does not shellcheck it: the rendered pod wrapper - begins with a leading blank line before ``#!/bin/bash`` (the template's - ``r\"\"\"`` opener), which is inert at runtime — the wrapper runs as - ``bash -c \"$script\"`` — but trips SC1128 as a standalone file. Keeping - the suffix lets the golden hold main's exact bytes (leading newline and - all) rather than a lint-friendly mutation of them. + The fixture deliberately carries the ``.sh.golden`` suffix (not + ``.sh``) so ``make lint-shell`` does not shellcheck it: the rendered + wrapper begins with a leading blank line before ``#!/bin/bash`` (the + template's ``r\"\"\"`` opener), which is inert at runtime — the + wrapper runs as ``bash -c \"$script\"`` — but trips SC1128 as a + standalone file. Keeping the suffix lets the golden hold the exact + bytes (leading newline and all). - To regenerate after an INTENTIONAL pod-path change, run from the - ``orchestrator/`` directory:: + To regenerate after an INTENTIONAL change, run from the repo root:: - python3 -c "from consensus_wrapper import \\ -build_consensus_wrapped_command as b; \\ -open('tests/golden/event_pump_wrapper_pod_default.sh.golden', 'w').write(b('x')[2])" + PYTHONPATH=orchestrator:shared .venv/bin/python -c \\ + "import consensus_wrapper as c; \\ +open('orchestrator/tests/golden/event_pump_wrapper.sh.golden','w')\\ +.write(c.build_consensus_wrapped_command('x')[2])" and review the diff — an unreviewed regeneration defeats the guard. """ - def test_pod_default_matches_golden_byte_for_byte(self, monkeypatch): - monkeypatch.delenv("EGG_EVENT_LOOP_OWNER", raising=False) - golden = _read_pod_default_golden() + def test_wrapper_matches_golden_byte_for_byte(self): + golden = _read_golden() cmd = build_consensus_wrapped_command("Prompt") assert cmd[:2] == ["bash", "-c"] assert cmd[2] == golden, ( - "pod-default wrapper rendering drifted from the golden " - "snapshot. If the change to the pod path was intentional, " - "regenerate tests/golden/event_pump_wrapper_pod_default.sh " - "(see this class's docstring) and review the diff; otherwise " - "this is the #3064 R1 guard catching an unintended drift of " - "the in-pod event-pump rendering." + "wrapper rendering drifted from the golden snapshot. If the " + "change to the wrapper was intentional, regenerate " + "tests/golden/event_pump_wrapper.sh.golden (see this class's " + "docstring) and review the diff; otherwise this is the " + "#3164 guard catching an unintended drift of the one-shot " + "wrapper rendering." ) - def test_explicit_pod_owner_matches_golden(self, monkeypatch): - """Explicit ``EGG_EVENT_LOOP_OWNER=pod`` is identical to unset.""" - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "pod") - assert build_consensus_wrapped_command("Prompt")[2] == _read_pod_default_golden() - - def test_pod_default_is_prompt_independent(self, monkeypatch): - """The event-pump emits its own per-event prompts, so the initial + def test_wrapper_is_prompt_independent(self): + """The wrapper emits its own per-event prompts, so the initial prompt text never reaches the bash — the golden must hold for any prompt argument.""" - monkeypatch.delenv("EGG_EVENT_LOOP_OWNER", raising=False) - golden = _read_pod_default_golden() + golden = _read_golden() assert build_consensus_wrapped_command("totally different prompt")[2] == golden - def test_golden_snapshot_is_the_in_pod_event_pump(self): - """Sanity-check the committed golden really is the in-pod - event-pump (blocking loop + wait-loop), so a stale/empty golden - can't make the byte-equality test vacuous.""" - golden = _read_pod_default_golden() - assert "Event-pump wrapper (#2908 slice-2)" in golden - assert "while true; do" in golden - assert "egg-orch message wait-loop" in golden - assert "start_background_heartbeat" in golden - - -class TestEventLoopOwnerAccessor: - """``env_config.get_event_loop_owner()`` — the #3064 slice-1 ownership - flag accessor. Unset/blank/whitespace falls back to the ``pod`` default - (the module's usual convention), but an unrecognised value is rejected - loudly by raising ``ValueError`` — the deliberate exception to the - never-raise convention, because there is no safe silent fallback for an - ownership mode (#3023: a wrong mode deadlocks BRC, so a typo must surface - at read time). See ``test_invalid_value_rejected_loudly``. - """ - - def test_unset_defaults_to_pod(self, monkeypatch): - monkeypatch.delenv("EGG_EVENT_LOOP_OWNER", raising=False) - from env_config import get_event_loop_owner - - assert get_event_loop_owner() == "pod" - - def test_explicit_pod(self, monkeypatch): - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "pod") - from env_config import get_event_loop_owner - - assert get_event_loop_owner() == "pod" - - def test_explicit_orchestrator(self, monkeypatch): - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "orchestrator") - from env_config import get_event_loop_owner - - assert get_event_loop_owner() == "orchestrator" - - def test_value_is_trimmed_and_case_insensitive(self, monkeypatch): - """Module convention (``.strip().lower()``): surrounding - whitespace and case must not change the parsed owner.""" - from env_config import get_event_loop_owner - - for raw in (" orchestrator", "orchestrator ", "Orchestrator", "ORCHESTRATOR"): - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", raw) - assert get_event_loop_owner() == "orchestrator", repr(raw) - for raw in (" pod ", "POD", "Pod"): - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", raw) - assert get_event_loop_owner() == "pod", repr(raw) - - def test_blank_value_defaults_to_pod(self, monkeypatch): - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", " ") - from env_config import get_event_loop_owner - - assert get_event_loop_owner() == "pod" - - def test_invalid_value_rejected_loudly(self, monkeypatch): - """An unrecognised ownership mode is rejected LOUDLY by raising - ``ValueError`` — there is no safe silent fallback for an ownership - mode (#3023: a wrong mode deadlocks BRC or duplicates pods, so a - typo must surface at read time rather than masquerade as the - default). This is the deliberate exception to the module's usual - never-raise/warn-and-default convention. The raised message must - name both the env var and the offending value so the operator can - find the typo.""" - from env_config import get_event_loop_owner - - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "kubernetes") - with pytest.raises(ValueError) as excinfo: - get_event_loop_owner() - msg = str(excinfo.value) - assert "EGG_EVENT_LOOP_OWNER" in msg and "kubernetes" in msg, ( - f"the ValueError must name the env var and the offending value. Got: {msg!r}" - ) + def test_wrapper_is_env_independent(self, monkeypatch): + """There is no ownership flag after #3164: any value of the + retired ``EGG_EVENT_LOOP_OWNER`` / ``EGG_BRC_EVENT_PUMP`` env vars + must produce the identical rendering.""" + golden = _read_golden() + for var in ("EGG_EVENT_LOOP_OWNER", "EGG_BRC_EVENT_PUMP"): + for val in ("pod", "orchestrator", "true", "false", "0"): + monkeypatch.setenv(var, val) + assert build_consensus_wrapped_command("Prompt")[2] == golden, ( + f"{var}={val!r} must not change the wrapper rendering " + "(no ownership flag after #3164)." + ) + monkeypatch.delenv(var, raising=False) + + def test_golden_snapshot_is_the_one_shot_wrapper(self): + """Sanity-check the committed golden really is the one-shot + wrapper (#3164), not the retired in-pod event-pump, so a + stale/empty golden can't make the byte-equality test vacuous.""" + golden = _read_golden() + # New one-shot markers. + assert "one-shot event handler (#3164)" in golden + assert 'ONE_SHOT_ACTION="${EGG_EVENT_ACTION:-}"' in golden + # Retired in-pod-pump markers MUST be gone. + assert "while true; do" not in golden + assert "egg-orch message wait-loop" not in golden + assert "start_background_heartbeat" not in golden + assert "# --- main event-pump loop ---" not in golden class TestOneShotArmStructure: - """Structural pins on the orchestrator-mode one-shot wrapper arm - (#3064 slice-1 task-1-2). With ``EGG_EVENT_LOOP_OWNER=orchestrator`` the - generated wrapper drops the in-pod blocking loop in favour of a - single-event arm driven by the injected ``EGG_EVENT_ACTION``. + """Structural pins on the one-shot wrapper (#3164). The wrapper is + one-shot only: it reads the injected ``EGG_EVENT_ACTION``, re-derives + next-action once as a stale-event backstop, optionally syncs, invokes + the agent once, and exits — no blocking loop, no background heartbeat. """ - def _one_shot_script(self, monkeypatch) -> str: - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "orchestrator") + def _script(self) -> str: cmd = build_consensus_wrapped_command("Prompt") assert cmd[:2] == ["bash", "-c"] return cmd[2] - def test_orchestrator_mode_differs_from_pod_default(self, monkeypatch): - monkeypatch.delenv("EGG_EVENT_LOOP_OWNER", raising=False) - pod = build_consensus_wrapped_command("Prompt")[2] - one_shot = self._one_shot_script(monkeypatch) - assert one_shot != pod, ( - "EGG_EVENT_LOOP_OWNER=orchestrator must select the one-shot " - "arm, which is NOT byte-identical to the in-pod event-pump." - ) - - @staticmethod - def _one_shot_arm_segment(script: str) -> str: - """Isolate the spliced one-shot arm — the region from where it - first reads the injected ``EGG_EVENT_LOOP_OWNER`` env up to the - main event-pump loop marker. The shared helper *definitions* - (``wait_for_event``, ``start_background_heartbeat``, …) live ABOVE - this region and the ``while true`` loop BELOW it, so this segment - is exactly the one-shot execution path — what task-1-2 calls "the - one-shot path".""" - main_marker = "# --- main event-pump loop ---" - assert main_marker in script, ( - "orchestrator-mode script must still contain the in-pod loop " - "(the one-shot arm reuses its helper functions and the loop is " - "the dormant fall-through when no event is injected)." - ) - owner_read = script.index("EGG_EVENT_LOOP_OWNER") - main_idx = script.index(main_marker) - assert owner_read < main_idx, ( - "the one-shot arm must be spliced BEFORE the main loop so an " - "injected event is handled and exits without ever reaching the " - "blocking loop." - ) - return script[owner_read:main_idx] - - def test_one_shot_path_does_not_block_on_wait_loop(self, monkeypatch): - """task-1-2: "absence of wait-loop … constructs in the one-shot - path". The arm's own block must not call the blocking bus wait — - the dedupe backstop is a single ``brc next-action`` re-check. (The - helper definitions remain in the script because the arm reuses - other helpers and the loop is the dormant fall-through; the arm - execution path simply never reaches them.)""" - arm = self._one_shot_arm_segment(self._one_shot_script(monkeypatch)) - assert "wait_for_event" not in arm, ( - "one-shot path must not call wait_for_event (blocking bus poll)." - ) - assert "egg-orch message wait-loop" not in arm, ( - "one-shot path must not block on the message bus." - ) - - def test_one_shot_path_starts_no_background_heartbeat(self, monkeypatch): - """task-1-2: "absence of … background-heartbeat constructs in the - one-shot path". A one-shot pod is short-lived, so the arm must not - start the wrapper-owned background heartbeat emitter (a single - foreground ``emit_heartbeat`` ping is fine and expected).""" - arm = self._one_shot_arm_segment(self._one_shot_script(monkeypatch)) - assert "start_background_heartbeat" not in arm, ( - "one-shot path must not spawn the background heartbeat emitter." - ) - - def test_one_shot_reads_injected_event_action(self, monkeypatch): - script = self._one_shot_script(monkeypatch) - assert "EGG_EVENT_ACTION" in script, ( - "one-shot arm is driven by the injected EGG_EVENT_ACTION env." - ) - - def test_one_shot_rechecks_next_action_once(self, monkeypatch): - script = self._one_shot_script(monkeypatch) - assert "brc next-action" in script, ( - "one-shot arm must re-check next-action once as the stale/dedupe backstop." - ) - - def test_one_shot_retains_agent_invocation_path(self, monkeypatch): - """The one-shot arm reuses the existing ``invoke_agent_for_event`` - composer path rather than inventing a parallel invocation.""" - script = self._one_shot_script(monkeypatch) + def test_has_no_main_event_pump_loop_marker(self): + """The retired in-pod pump's ``# --- main event-pump loop ---`` + marker and its ``while true`` loop must be gone.""" + script = self._script() + assert "# --- main event-pump loop ---" not in script + assert "while true; do" not in script + + def test_does_not_block_on_wait_loop(self): + """The wrapper never blocks on the bus: there is no + ``wait_for_event`` / ``egg-orch message wait-loop`` call. The + stale-event backstop is a single ``brc next-action`` re-check.""" + script = self._script() + assert "wait_for_event" not in script, ( + "one-shot wrapper must not call wait_for_event (blocking bus poll)." + ) + assert "egg-orch message wait-loop" not in script, ( + "one-shot wrapper must not block on the message bus." + ) + + def test_starts_no_background_heartbeat(self): + """A one-shot pod is short-lived, so the wrapper must not start a + background heartbeat emitter (a single foreground + ``emit_heartbeat`` ping is fine and expected).""" + script = self._script() + assert "start_background_heartbeat" not in script, ( + "one-shot wrapper must not spawn the background heartbeat emitter." + ) + # A single foreground ping is still expected. + assert 'emit_heartbeat "WORKING"' in script + + def test_reads_injected_event_action(self): + script = self._script() + assert 'ONE_SHOT_ACTION="${EGG_EVENT_ACTION:-}"' in script, ( + "one-shot wrapper is driven by the injected EGG_EVENT_ACTION env." + ) + + def test_has_backstop_exit_paths(self): + """The stale-event backstop exit codes are pinned: + - exit 64 (EX_USAGE) on a missing / terminal / unknown action; + - exit 75 (EX_TEMPFAIL) on an inconclusive next-action re-check; + - exit 0 on a confirmed-stale event (no agent invocation).""" + script = self._script() + assert "exit 64" in script + assert "exit 75" in script + assert "exit 0" in script + + def test_rechecks_next_action_once(self): + script = self._script() + assert "fetch_next_action" in script, ( + "one-shot wrapper must re-derive next-action once as the stale/dedupe backstop." + ) + + def test_retains_agent_invocation_path(self): + """The one-shot wrapper reuses the existing + ``invoke_agent_for_event`` composer path and ``sync_to_proposals`` + rather than inventing a parallel invocation.""" + script = self._script() assert "invoke_agent_for_event" in script + assert "sync_to_proposals" in script class TestOneShotArmBehavior: - """Behavioural tests of the orchestrator-mode one-shot arm, driving the - rendered bash against PATH stubs (#3064 slice-1 task-1-2). Covers the - one-shot behaviors: confirmed-stale exit 0, inconclusive-recheck exit 75 - (EX_TEMPFAIL), exactly-one-invocation, agent exit-code passthrough, and - loud rejection of injected confirm/complete. + """Behavioural tests of the one-shot wrapper, driving the rendered + bash against PATH stubs (#3164). Covers the one-shot behaviors: + confirmed-stale exit 0, inconclusive-recheck exit 75 (EX_TEMPFAIL), + exactly-one-invocation, agent exit-code passthrough, and loud + rejection of injected confirm/complete. """ - def _render(self, monkeypatch) -> list[str]: - monkeypatch.setenv("EGG_EVENT_LOOP_OWNER", "orchestrator") + def _render(self) -> list[str]: return build_consensus_wrapped_command("Prompt") @staticmethod @@ -2491,9 +1011,8 @@ def _env(bin_dir, action: str, dedupe: str = "evt-1"): env = os.environ.copy() env["PATH"] = f"{bin_dir}:{env.get('PATH', '')}" env["EGG_AGENT_ROLE"] = "tester" - env["EGG_PIPELINE_ID"] = "issue-3064" + env["EGG_PIPELINE_ID"] = "issue-3164" env["EGG_SLICE_ID"] = "slice-1" - env["EGG_EVENT_LOOP_OWNER"] = "orchestrator" env["EGG_EVENT_ACTION"] = action env["EGG_EVENT_DEDUPE_KEY"] = dedupe # Force the compose_event_prompt fallback (unreadable default path) @@ -2507,13 +1026,13 @@ def _agent_invocation_count(agent_log) -> int: return 0 return len([ln for ln in agent_log.read_text().splitlines() if ln.strip()]) - def test_stale_event_exits_zero_without_invoking_agent(self, tmp_path, monkeypatch): + def test_stale_event_exits_zero_without_invoking_agent(self, tmp_path): """The orchestrator injected a ``propose`` event, but by the time the pod starts the live next-action has moved to ``wait`` (another - pod already handled it). The one-shot arm must re-check, detect the - stale event, and exit 0 WITHOUT invoking the agent — the dedupe + pod already handled it). The one-shot wrapper must re-check, detect + the stale event, and exit 0 WITHOUT invoking the agent — the dedupe backstop.""" - cmd = self._render(monkeypatch) + cmd = self._render() bin_dir, general_log, agent_log = self._stub_bin( tmp_path, '{"action":"wait"}', agent_exit=0 ) @@ -2526,16 +1045,15 @@ def test_stale_event_exits_zero_without_invoking_agent(self, tmp_path, monkeypat + (general_log.read_text() if general_log.exists() else "(empty)") ) - def test_inconclusive_recheck_exits_ex_tempfail_without_invoking(self, tmp_path, monkeypatch): + def test_inconclusive_recheck_exits_ex_tempfail_without_invoking(self, tmp_path): """A transient blip at re-check time (``egg-orch brc next-action`` returns non-zero — 409 / 5xx / transport) makes ``fetch_next_action`` - fall back to ``{"action":"wait"}``. The arm must NOT report a clean - handoff (exit 0) — that would let a transient failure silently drop a - live event — and must NOT invoke the agent. Instead it exits 75 - (EX_TEMPFAIL) so the slice-3 supervisor re-derives next-action itself - rather than treating the event as definitively handled (reviewer - non-blocking note, PR #3167).""" - cmd = self._render(monkeypatch) + fall back to ``{"action":"wait"}``. The wrapper must NOT report a + clean handoff (exit 0) — that would let a transient failure silently + drop a live event — and must NOT invoke the agent. Instead it exits + 75 (EX_TEMPFAIL) so the orchestrator supervisor re-derives next-action + itself rather than treating the event as definitively handled.""" + cmd = self._render() bin_dir, general_log, agent_log = self._stub_bin( tmp_path, '{"action":"propose"}', agent_exit=0, next_action_rc=22 ) @@ -2551,11 +1069,11 @@ def test_inconclusive_recheck_exits_ex_tempfail_without_invoking(self, tmp_path, + (general_log.read_text() if general_log.exists() else "(empty)") ) - def test_fresh_event_invokes_agent_exactly_once(self, tmp_path, monkeypatch): + def test_fresh_event_invokes_agent_exactly_once(self, tmp_path): """A fresh ``propose`` (live next-action still ``propose``) invokes the agent exactly once, then exits — no loop, no second invocation.""" - cmd = self._render(monkeypatch) + cmd = self._render() bin_dir, general_log, agent_log = self._stub_bin( tmp_path, '{"action":"propose"}', agent_exit=0 ) @@ -2571,12 +1089,12 @@ def test_fresh_event_invokes_agent_exactly_once(self, tmp_path, monkeypatch): + (general_log.read_text() if general_log.exists() else "(empty)") ) - def test_agent_exit_code_is_passed_through(self, tmp_path, monkeypatch): + def test_agent_exit_code_is_passed_through(self, tmp_path): """#2908 exit-code classification passthrough: a non-zero agent exit (17 — not a signal code, so not reclassified) propagates to the wrapper's exit code rather than being swallowed, so the - orchestrator can supervise the failure (slice-3).""" - cmd = self._render(monkeypatch) + orchestrator can supervise the failure.""" + cmd = self._render() bin_dir, general_log, agent_log = self._stub_bin( tmp_path, '{"action":"propose"}', agent_exit=17 ) @@ -2586,40 +1104,39 @@ def test_agent_exit_code_is_passed_through(self, tmp_path, monkeypatch): "agent should have been invoked once before the failure." ) assert result.returncode == 17, ( - "one-shot arm must pass the agent's exit code through (not " + "one-shot wrapper must pass the agent's exit code through (not " f"swallow it to 0/1); got {result.returncode}. stderr:\n" + result.stderr ) - def test_injected_confirm_is_rejected_loudly(self, tmp_path, monkeypatch): - self._assert_terminal_action_rejected(tmp_path, monkeypatch, "confirm") + def test_injected_confirm_is_rejected_loudly(self, tmp_path): + self._assert_terminal_action_rejected(tmp_path, "confirm") - def test_injected_complete_is_rejected_loudly(self, tmp_path, monkeypatch): - self._assert_terminal_action_rejected(tmp_path, monkeypatch, "complete") + def test_injected_complete_is_rejected_loudly(self, tmp_path): + self._assert_terminal_action_rejected(tmp_path, "complete") - def _assert_terminal_action_rejected(self, tmp_path, monkeypatch, action: str): + def _assert_terminal_action_rejected(self, tmp_path, action: str): """``confirm``/``complete`` are executed orchestrator-side - (agent-free) under ``EGG_EVENT_LOOP_OWNER=orchestrator``; a pod is - never spawned for them. If one is injected anyway, the one-shot arm - must reject it loudly: non-zero exit, an error on stderr, no agent - invocation, and no ``consensus confirmed`` call.""" - cmd = self._render(monkeypatch) + (agent-free); a pod is never spawned for them. If one is injected + anyway, the one-shot wrapper must reject it loudly: non-zero exit, + an error on stderr, no agent invocation, and no ``consensus + confirmed`` call.""" + cmd = self._render() bin_dir, general_log, agent_log = self._stub_bin( tmp_path, f'{{"action":"{action}"}}', agent_exit=0 ) env = self._env(bin_dir, action) result = subprocess.run(cmd, env=env, capture_output=True, text=True, timeout=30) assert result.returncode != 0, ( - f"one-shot arm must reject an injected {action!r} action with a " + f"one-shot wrapper must reject an injected {action!r} action with a " f"non-zero exit. stderr:\n{result.stderr}" ) assert self._agent_invocation_count(agent_log) == 0, ( - f"one-shot arm must NOT invoke the agent for {action!r}." + f"one-shot wrapper must NOT invoke the agent for {action!r}." ) log = general_log.read_text() if general_log.exists() else "" assert "consensus confirmed" not in log, ( - f"one-shot arm must NOT run 'consensus confirmed' for {action!r}; " - "confirm/complete are orchestrator-side in owner=orchestrator " - f"mode. egg-orch log:\n{log}" + f"one-shot wrapper must NOT run 'consensus confirmed' for {action!r}; " + f"confirm/complete are orchestrator-side. egg-orch log:\n{log}" ) assert action in result.stderr, ( f"rejection of {action!r} must be LOUD — an error naming the " diff --git a/orchestrator/tests/test_event_loop.py b/orchestrator/tests/test_event_loop.py index 8c9256ce35..259f48db8e 100644 --- a/orchestrator/tests/test_event_loop.py +++ b/orchestrator/tests/test_event_loop.py @@ -868,43 +868,13 @@ def test_loop_reexports_equal_supervision_policy(self): == supervision_policy.SUPERVISION_FAILURE_STREAK_ALERT ) - def test_wrapper_template_renders_supervision_policy_values(self): - """The rendered pod wrapper embeds the supervision_policy constants. - - Slice-3 AC4: the wrapper template and the loop must read identical - constants. ``consensus_wrapper`` re-exports them from - ``supervision_policy`` and interpolates them into the bash template, - so the rendered guards/backoff must reflect the single-source values. - A fork (hardcoded literal that diverges from supervision_policy) makes - one of these substrings disappear from the render. - """ - import supervision_policy - from consensus_wrapper import ( - SUPERVISION_BACKOFF_CAP_SECONDS, - SUPERVISION_BACKOFF_FACTOR, - SUPERVISION_FAILURE_STREAK_ALERT, - SUPERVISION_FAILURE_STREAK_WARN, - build_consensus_wrapped_command, - ) - - # The wrapper's re-exports are the same single source as the loop's. - assert SUPERVISION_BACKOFF_FACTOR == supervision_policy.SUPERVISION_BACKOFF_FACTOR - assert SUPERVISION_BACKOFF_CAP_SECONDS == supervision_policy.SUPERVISION_BACKOFF_CAP_SECONDS - assert SUPERVISION_FAILURE_STREAK_WARN == supervision_policy.SUPERVISION_FAILURE_STREAK_WARN - assert ( - SUPERVISION_FAILURE_STREAK_ALERT == supervision_policy.SUPERVISION_FAILURE_STREAK_ALERT - ) - - script = build_consensus_wrapped_command("x")[2] - warn = supervision_policy.SUPERVISION_FAILURE_STREAK_WARN - alert = supervision_policy.SUPERVISION_FAILURE_STREAK_ALERT - factor = supervision_policy.SUPERVISION_BACKOFF_FACTOR - cap = supervision_policy.SUPERVISION_BACKOFF_CAP_SECONDS - # Guards interpolate the constants verbatim into the bash template. - assert f'-ge {warn} ] && [ "$AGENT_FAIL_ALERTED_5"' in script - assert f'-ge {alert} ] && [ "$AGENT_FAIL_ALERTED_10"' in script - assert f"AGENT_FAIL_STREAK * {factor}" in script - assert f'"$agent_backoff_secs" -gt {cap}' in script + # #3164 retired ``test_wrapper_template_renders_supervision_policy_values``: + # the failure-streak backoff/escalation machinery was deleted from the + # in-pod wrapper template (the orchestrator event loop's ``JobSupervisor`` + # owns it now). The wrapper no longer interpolates the supervision_policy + # constants, so the loop's re-export equality + # (``test_loop_reexports_equal_supervision_policy`` above) is the remaining + # single-source guard. # --------------------------------------------------------------------------- diff --git a/orchestrator/tests/test_health_monitor.py b/orchestrator/tests/test_health_monitor.py index ec9e057591..285991d488 100644 --- a/orchestrator/tests/test_health_monitor.py +++ b/orchestrator/tests/test_health_monitor.py @@ -169,6 +169,7 @@ def test_heartbeat_timeout_triggers_escalation(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -185,6 +186,7 @@ def test_heartbeat_within_threshold_no_alert(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=120) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -232,6 +234,7 @@ def test_recent_activity_suppresses_heartbeat_alert(self): orchestrator_activity_quiet_seconds=120, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -263,6 +266,7 @@ def test_recent_activity_suppresses_progress_alert(self): orchestrator_activity_quiet_seconds=120, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -283,6 +287,7 @@ def test_stale_activity_does_not_suppress(self): orchestrator_activity_quiet_seconds=120, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) _emit_container_activity(bus, agent_id=AGENT_ID) @@ -304,6 +309,7 @@ def test_no_activity_event_does_not_suppress(self): orchestrator_activity_quiet_seconds=120, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -326,6 +332,7 @@ def test_disabled_gate_no_suppression(self): orchestrator_activity_quiet_seconds=0, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -348,6 +355,7 @@ def test_activity_event_for_other_pipeline_ignored(self): orchestrator_activity_quiet_seconds=120, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) _emit_container_activity(bus, agent_id=AGENT_ID, pipeline_id="some-other-pipeline") @@ -374,6 +382,7 @@ def test_activity_event_for_other_agent_does_not_suppress_focal(self): orchestrator_alert_progress_gate_seconds=0, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) _emit_heartbeat(bus, agent_id=AGENT_ID) _emit_heartbeat(bus, agent_id=AGENT_ID_2) @@ -402,6 +411,7 @@ def test_reset_clears_heartbeat_anchor(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) assert AGENT_ID in monitor._last_heartbeat @@ -416,6 +426,9 @@ def test_reset_clears_heartbeat_anchor(self): # Reset and verify the next check (with no fresh heartbeat) does not # synthesize an alert from the dead anchor. monitor.reset_agent(AGENT_ID) + # The respawned role no longer has a live one-shot Job until the loop + # re-spawns it, so it drops out of the active set too. + monitor.set_active_roles(set()) assert AGENT_ID not in monitor._last_heartbeat assert AGENT_ID not in monitor._agents @@ -467,6 +480,8 @@ def test_container_exit_creates_hitl_escalation(self): bus = _make_event_bus() monitor = _make_monitor(bus) + monitor.set_active_roles({AGENT_ID}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -483,6 +498,8 @@ def test_container_exit_zero_still_escalates(self): bus = _make_event_bus() monitor = _make_monitor(bus) + monitor.set_active_roles({AGENT_ID}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -600,6 +617,7 @@ def test_progress_stall_triggers_escalation(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -616,6 +634,7 @@ def test_progress_stall_resolved_no_second_escalation(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -644,6 +663,7 @@ def test_progress_escalation_deduplication(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -674,6 +694,7 @@ def test_progress_re_escalates_after_recovery(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -747,6 +768,8 @@ def test_container_exit_always_hitl(self): config = _make_config(overseer_enabled=True) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -783,6 +806,7 @@ def test_resolved_alerts_removed(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) # Register agent _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -866,6 +890,7 @@ def test_heartbeat_timeout_per_agent(self): orchestrator_alert_progress_gate_seconds=0, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) # Both agents send heartbeats _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -925,6 +950,7 @@ def test_heartbeat_timeout_fires_escalation_callback(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -943,6 +969,7 @@ def test_progress_stall_fires_escalation_callback(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) @@ -961,6 +988,7 @@ def test_heartbeat_no_re_escalation_on_subsequent_cycles(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -985,6 +1013,7 @@ def test_heartbeat_escalation_resets_on_heartbeat(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1026,6 +1055,7 @@ def test_resolve_alerts_clears_matching(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1131,6 +1161,7 @@ def test_implement_phase_uses_higher_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) monitor.set_current_phase("implement") _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1152,6 +1183,7 @@ def test_implement_phase_escalates_beyond_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) monitor.set_current_phase("implement") _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1172,6 +1204,7 @@ def test_non_implement_phase_uses_default_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) monitor.set_current_phase("plan") _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1191,6 +1224,7 @@ def test_no_phase_set_uses_default_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) # No set_current_phase call _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -1209,6 +1243,7 @@ def test_implement_phase_progress_stall_uses_higher_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) monitor.set_current_phase("implement") _emit_progress(bus, agent_id=AGENT_ID) @@ -1227,6 +1262,7 @@ def test_implement_phase_progress_stall_escalates_beyond_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) monitor.set_current_phase("implement") _emit_progress(bus, agent_id=AGENT_ID) @@ -1246,6 +1282,7 @@ def test_phase_transition_changes_threshold(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) # Start in implement phase - no escalation at 200s monitor.set_current_phase("implement") @@ -1357,6 +1394,8 @@ def test_reviewer_only_suppressed_when_producers_working(self): config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({REVIEWER_ID}) + mock_tracker = self._make_mock_tracker( reviewer_roles=[REVIEWER_ID], producer_roles=[PRODUCER_ID], @@ -1398,6 +1437,7 @@ def test_reviewer_not_suppressed_when_producer_proposed(self): proposal_timestamps={PRODUCER_ID: now - 400}, # well past 300s grace ) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1428,6 +1468,7 @@ def test_dual_role_agent_not_suppressed(self): producers_for_reviewer={dual_role_id: [PRODUCER_ID]}, ) + monitor.set_active_roles({dual_role_id}) _emit_heartbeat(bus, agent_id=dual_role_id) with ( @@ -1457,6 +1498,7 @@ def test_brc_suppression_on_progress_check(self): producers_for_reviewer={REVIEWER_ID: [PRODUCER_ID]}, ) + monitor.set_active_roles({REVIEWER_ID}) _emit_progress(bus, agent_id=REVIEWER_ID) with ( @@ -1476,6 +1518,7 @@ def test_no_tracker_means_no_suppression(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) @@ -1515,6 +1558,7 @@ def test_reviewer_with_multiple_producers_partial_proposed(self): proposal_timestamps={PRODUCER_ID: now - 400}, # past grace ) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1550,6 +1594,7 @@ def test_reviewer_with_multiple_producers_all_working(self): producers_for_reviewer={REVIEWER_ID: [PRODUCER_ID, producer_2]}, ) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1579,6 +1624,7 @@ def test_producer_not_suppressed_even_when_working(self): producers_for_reviewer={REVIEWER_ID: [PRODUCER_ID]}, ) + monitor.set_active_roles({PRODUCER_ID}) _emit_heartbeat(bus, agent_id=PRODUCER_ID) with ( @@ -1614,6 +1660,7 @@ def test_brc_suppression_resumes_monitoring_after_proposal(self): producers_for_reviewer={REVIEWER_ID: [PRODUCER_ID]}, ) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1655,6 +1702,7 @@ def test_implement_phase_with_brc_suppression(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({PRODUCER_ID, REVIEWER_ID}) monitor.set_current_phase("implement") mock_tracker = _make_mock_tracker( @@ -1688,6 +1736,7 @@ def test_implement_producer_exceeds_threshold_reviewer_idle(self): orchestrator_implement_heartbeat_timeout_seconds=600, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({PRODUCER_ID, REVIEWER_ID}) monitor.set_current_phase("implement") mock_tracker = _make_mock_tracker( @@ -1761,6 +1810,7 @@ def test_reviewer_within_grace_suppressed_heartbeat(self): # Producer proposed 100s ago — within 300s grace mock_tracker = self._make_tracker_with_proposal(proposal_epoch=now - 100) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1787,6 +1837,7 @@ def test_reviewer_within_grace_suppressed_progress(self): now = time.time() mock_tracker = self._make_tracker_with_proposal(proposal_epoch=now - 100) + monitor.set_active_roles({REVIEWER_ID}) _emit_progress(bus, agent_id=REVIEWER_ID) with ( @@ -1814,6 +1865,7 @@ def test_reviewer_past_grace_is_flagged(self): # Producer proposed 400s ago — past 300s grace mock_tracker = self._make_tracker_with_proposal(proposal_epoch=now - 400) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1841,6 +1893,7 @@ def test_reviewer_past_grace_flagged_progress(self): now = time.time() mock_tracker = self._make_tracker_with_proposal(proposal_epoch=now - 400) + monitor.set_active_roles({REVIEWER_ID}) _emit_progress(bus, agent_id=REVIEWER_ID) with ( @@ -1873,6 +1926,7 @@ def test_grace_boundary_exactly_at_threshold(self): proposal_epoch = check_time - grace mock_tracker = self._make_tracker_with_proposal(proposal_epoch=proposal_epoch) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1901,6 +1955,7 @@ def test_custom_grace_period_config(self): # Proposed 80s ago — past custom 60s grace mock_tracker = self._make_tracker_with_proposal(proposal_epoch=now - 80) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1935,6 +1990,7 @@ def test_no_proposals_means_not_suppressed(self): # No proposal_timestamps → returns None ) + monitor.set_active_roles({REVIEWER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) with ( @@ -1971,6 +2027,7 @@ def test_dual_role_not_suppressed_by_grace(self): proposal_timestamps={PRODUCER_ID: now - 10}, ) + monitor.set_active_roles({dual_id}) _emit_heartbeat(bus, agent_id=dual_id) with ( @@ -2851,6 +2908,7 @@ def test_reviewer_suppressed_while_producer_past_ack_timeout(self): escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) + monitor.set_active_roles({REVIEWER_ID, PRODUCER_ID}) _emit_heartbeat(bus, agent_id=REVIEWER_ID) _emit_heartbeat(bus, agent_id=PRODUCER_ID) @@ -2906,6 +2964,7 @@ def test_heartbeat_message_resets_last_heartbeat(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) # Prime the agent state via a legacy heartbeat so the agent is # tracked. @@ -2935,6 +2994,7 @@ def test_non_heartbeat_message_does_not_reset(self): bus = _make_event_bus() config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -2963,6 +3023,8 @@ def test_heartbeat_message_clears_escalation_flag(self): config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) + # 1) agent is tracked. _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3002,6 +3064,8 @@ def test_heartbeat_event_accepts_from_role_alias(self): config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + monitor.set_active_roles({"coder"}) + # First tracking via legacy path so an entry exists. _emit_heartbeat(bus, agent_id="coder") @@ -3045,6 +3109,7 @@ def test_heartbeat_alert_defers_when_peer_heartbeat_fresh(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) base = time.time() @@ -3077,6 +3142,7 @@ def test_heartbeat_alert_fires_when_gate_window_elapsed(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3098,6 +3164,7 @@ def test_heartbeat_alert_excludes_self_from_peer_signal(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3119,6 +3186,7 @@ def test_gate_disabled_when_seconds_zero(self): orchestrator_alert_progress_gate_seconds=0, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3143,6 +3211,7 @@ def test_progress_alert_defers_when_peer_heartbeat_fresh(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) base = time.time() _emit_progress(bus, agent_id=AGENT_ID) @@ -3169,6 +3238,7 @@ def test_gate_defers_when_brc_bus_active(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3200,6 +3270,7 @@ def test_gate_handles_missing_tracker_gracefully(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3224,6 +3295,8 @@ def test_gate_filters_inactive_agent_heartbeats(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + # Only AGENT_ID has a live Job; AGENT_ID_2 lingers from a prior phase. + monitor.set_active_roles({AGENT_ID}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3280,6 +3353,7 @@ def test_overseer_heartbeat_alert_fires_despite_peer_heartbeats(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({"overseer", AGENT_ID, AGENT_ID_2}) base = time.time() @@ -3312,6 +3386,7 @@ def test_overseer_progress_alert_fires_despite_peer_heartbeats(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({"overseer", AGENT_ID}) base = time.time() _emit_progress(bus, agent_id="overseer") @@ -3342,6 +3417,7 @@ def test_overseer_silence_routed_to_hitl_not_self(self): ) monitor = _make_monitor(bus, config) monitor.on_escalation(escalations.append) + monitor.set_active_roles({"overseer"}) base = time.time() _emit_heartbeat(bus, agent_id="overseer") @@ -3364,6 +3440,7 @@ def test_non_overseer_silence_still_deferred_by_peers(self): orchestrator_alert_progress_gate_seconds=300, ) monitor = _make_monitor(bus, config) + monitor.set_active_roles({AGENT_ID, AGENT_ID_2}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3392,6 +3469,7 @@ def test_non_overseer_escalation_still_routes_to_overseer(self): ) monitor = _make_monitor(bus, config) monitor.on_escalation(escalations.append) + monitor.set_active_roles({AGENT_ID}) base = time.time() _emit_heartbeat(bus, agent_id=AGENT_ID) @@ -3544,33 +3622,26 @@ def test_check_brc_progress_uses_plan_phase_threshold(self): # --------------------------------------------------------------------------- -# Slice-5: ownership-mode awareness (#3064 TASK-5-2) +# Active-Job-scoped tripwires (#3064 slice-5 / #3164) # --------------------------------------------------------------------------- -# These tests are written test-first: they pin the slice-5 contract that the -# coder's TASK-5-1 must satisfy. Per the plan (slice-5) the health monitor -# becomes ownership-mode-aware: -# -# * Orchestrator mode: a role with no active Job is normal — never alerts -# on heartbeat/progress timeout; heartbeat-timeout and container-exit -# tripwires apply ONLY while a one-shot Job is active for that role; a -# silent mid-event pod (Job alive, no heartbeats) still trips. -# * Pod mode (default): existing behavior unchanged. -# -# The monitor reads the ownership mode from ``env_config.get_event_loop_owner()`` -# (already landed in slice-1, task-1-1). Active-role awareness is injected via -# ``set_active_roles()`` — the event loop (slice-2/3/4) or concurrent executor -# calls this to tell the monitor which roles currently have a spawned Job. -# -# These tests remain RED until the coder lands TASK-5-1. +# The heartbeat/progress/container-exit tripwires are gated unconditionally on +# the active one-shot-Job set (#3164 retired the in-pod ownership mode): a role +# is checked ONLY while a one-shot Job is active for it. With an EMPTY active +# set (the default for a fresh monitor) ALL tripwires are skipped — a role with +# no active Job is legitimately idle. A silent mid-event pod (Job in the active +# set, no heartbeats) still trips. Active-role awareness is injected via +# ``set_active_roles()`` — the event loop or concurrent executor calls this each +# poll to tell the monitor which roles currently have a spawned Job. # --------------------------------------------------------------------------- class TestOwnershipModeHeartbeatMatrix: - """Ownership-mode matrix for heartbeat timeout tripwire (#3064 slice-5). + """Active-Job-scoped tripwire matrix (#3064 slice-5 / #3164). - Both ownership modes asserted side by side per tripwire scenario: - orchestrator: no-pod -> no alarm, active-Job-only scoping, - silent mid-event pod still trips. Pod mode: unchanged. + Tripwire scoping is unconditional now that the in-pod ownership mode is + retired: a role with no active Job never alarms; tripwires apply ONLY + while a one-shot Job is active for that role; a silent mid-event pod + (Job alive, no heartbeats) still trips. """ PIPELINE = "issue-3064-slice-5-hb" @@ -3581,35 +3652,33 @@ def _orchestrator_role(self) -> str: return "coder" def test_orchestrator_mode_no_pod_no_alert(self): - """Orchestrator mode: a role with no active Job never alerts.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + """A role with no active Job never alerts.""" + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - _emit_heartbeat(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) + _emit_heartbeat(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_heartbeats() # No alert — no active Job for this role. assert actions == [] def test_orchestrator_mode_active_job_triggers_alert(self): - """Orchestrator mode: a role with an active Job alerts on timeout.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + """A role with an active Job alerts on timeout.""" + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - # Register the role as having an active Job. - monitor.set_active_roles({self._orchestrator_role}) - _emit_heartbeat(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) + # Register the role as having an active Job. + monitor.set_active_roles({self._orchestrator_role}) + _emit_heartbeat(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_heartbeats() assert len(actions) == 1 assert actions[0]["agent_id"] == self._orchestrator_role @@ -3622,120 +3691,77 @@ def test_orchestrator_mode_silent_mid_event_pod_still_trips(self): heartbeats should still trip the heartbeat timeout — the Job is running but its agent has stalled. """ - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - # Register the role as having an active Job but never emit a - # heartbeat — simulating a pod that was spawned but the agent - # inside never started. - monitor.set_active_roles({self._orchestrator_role}) + # Register the role as having an active Job but never emit a + # heartbeat — simulating a pod that was spawned but the agent + # inside never started. + monitor.set_active_roles({self._orchestrator_role}) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_heartbeats() # The agent is registered as active but has no heartbeat — trips alert. assert len(actions) == 1 assert actions[0]["agent_id"] == self._orchestrator_role def test_orchestrator_mode_active_job_container_exit_still_trips(self): - """Orchestrator mode: a container exit for a role with an active Job - escalates — a live one-shot pod died mid-event. + """A container exit for a role with an active Job escalates — a live + one-shot pod died mid-event. """ - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config() - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - - # Role has a live Job — its container dying is a real failure. - monitor.set_active_roles({self._orchestrator_role}) - _emit_container_stopped( - bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE, exit_code=137 - ) + bus = _make_event_bus() + config = _make_config() + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + + # Role has a live Job — its container dying is a real failure. + monitor.set_active_roles({self._orchestrator_role}) + _emit_container_stopped( + bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE, exit_code=137 + ) - monitor.check_tripwires() + monitor.check_tripwires() alerts = [a for a in monitor.get_active_alerts() if a.get("alert_type") == "container_exit"] assert len(alerts) == 1 def test_orchestrator_mode_ghost_container_exit_suppressed(self): - """Orchestrator mode: a container exit for a role with NO active Job - is a ghost (a one-shot pod that finished its event and was reaped) - and must NOT escalate — the criterion scopes container-exit - tripwires to roles with a live Job, consistent with the heartbeat - path's empty-active-set suppression. + """A container exit for a role with NO active Job is a ghost (a + one-shot pod that finished its event and was reaped) and must NOT + escalate — the criterion scopes container-exit tripwires to roles + with a live Job, consistent with the heartbeat path's empty-active-set + suppression. """ - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config() - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - - # No active Job for this role — the exit is an expected reap. - _emit_container_stopped( - bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE, exit_code=0 - ) - - monitor.check_tripwires() - - alerts = [a for a in monitor.get_active_alerts() if a.get("alert_type") == "container_exit"] - assert alerts == [] + bus = _make_event_bus() + config = _make_config() + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - def test_pod_mode_container_exit_always_trips(self): - """Pod mode: a container exit always escalates, active-role set - ignored (no one-shot Jobs in pod mode). - """ - with patch("env_config.get_event_loop_owner", return_value="pod"): - bus = _make_event_bus() - config = _make_config() - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - - # Empty active set is meaningless in pod mode — exit still trips. - monitor.set_active_roles(set()) - _emit_container_stopped( - bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE, exit_code=137 - ) + # No active Job for this role — the exit is an expected reap. + _emit_container_stopped( + bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE, exit_code=0 + ) - monitor.check_tripwires() + monitor.check_tripwires() alerts = [a for a in monitor.get_active_alerts() if a.get("alert_type") == "container_exit"] - assert len(alerts) == 1 - - def test_pod_mode_unchanged(self): - """Pod mode: heartbeat timeout fires regardless of active-role set.""" - with patch("env_config.get_event_loop_owner", return_value="pod"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - - # Even with empty active roles (which is meaningless in pod mode), - # the monitor ignores the set and fires normally. - monitor.set_active_roles(set()) - _emit_heartbeat(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) - - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() - - assert len(actions) == 1 - assert actions[0]["agent_id"] == self._orchestrator_role + assert alerts == [] def test_orchestrator_mode_multiple_roles_independent(self): """Active-role scoping is per-role — mixed set has mixed outcomes.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - # Only "coder" has an active Job; "tester" does not. - monitor.set_active_roles({"coder"}) - for role in ("coder", "tester"): - _emit_heartbeat(bus, agent_id=role, pipeline_id=self.PIPELINE) + # Only "coder" has an active Job; "tester" does not. + monitor.set_active_roles({"coder"}) + for role in ("coder", "tester"): + _emit_heartbeat(bus, agent_id=role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_heartbeats() # Only coder (active Job) should alert; tester (no Job) should not. assert len(actions) == 1 @@ -3743,77 +3769,54 @@ def test_orchestrator_mode_multiple_roles_independent(self): def test_orchestrator_mode_progress_stall_no_pod_no_alert(self): """Progress stall is also suppressed when no active Job exists.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - _emit_progress(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) + _emit_progress(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_progress() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_progress() assert actions == [] def test_orchestrator_mode_progress_stall_active_job_triggers(self): """Progress stall fires when role has an active Job.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - monitor.set_active_roles({self._orchestrator_role}) - _emit_progress(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) + monitor.set_active_roles({self._orchestrator_role}) + _emit_progress(bus, agent_id=self._orchestrator_role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_progress() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_progress() assert len(actions) == 1 assert actions[0]["agent_id"] == self._orchestrator_role def test_set_active_roles_replace_not_accumulate(self): """set_active_roles replaces the prior set, it does not accumulate.""" - with patch("env_config.get_event_loop_owner", return_value="orchestrator"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) + bus = _make_event_bus() + config = _make_config(orchestrator_heartbeat_timeout_seconds=60) + monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - monitor.set_active_roles({"coder", "tester"}) - # Second call replaces, removing "tester". - monitor.set_active_roles({"coder"}) + monitor.set_active_roles({"coder", "tester"}) + # Second call replaces, removing "tester". + monitor.set_active_roles({"coder"}) - for role in ("coder", "tester"): - _emit_heartbeat(bus, agent_id=role, pipeline_id=self.PIPELINE) + for role in ("coder", "tester"): + _emit_heartbeat(bus, agent_id=role, pipeline_id=self.PIPELINE) - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() + with patch("health_monitor.time") as mock_time: + mock_time.time.return_value = time.time() + 300 + actions = monitor.check_heartbeats() assert len(actions) == 1 assert actions[0]["agent_id"] == "coder" - def test_pod_mode_active_roles_set_ignored(self): - """In pod mode, the active-role set is ignored; all agents alert.""" - with patch("env_config.get_event_loop_owner", return_value="pod"): - bus = _make_event_bus() - config = _make_config(orchestrator_heartbeat_timeout_seconds=60) - monitor = _make_monitor(bus, config, pipeline_id=self.PIPELINE) - - # Set active roles to a small subset — should be ignored in pod mode. - monitor.set_active_roles({"tester"}) - for role in ("coder", "tester"): - _emit_heartbeat(bus, agent_id=role, pipeline_id=self.PIPELINE) - - with patch("health_monitor.time") as mock_time: - mock_time.time.return_value = time.time() + 300 - actions = monitor.check_heartbeats() - - # Both agents alert despite active_roles only containing "tester". - agent_ids = {a["agent_id"] for a in actions} - assert agent_ids == {"coder", "tester"} - class TestOwnershipModeIdleBudgetAnomaly: """Idle-budget anomaly-name equality (#3064 slice-5 TASK-5-2). diff --git a/orchestrator/tests/test_heartbeat.py b/orchestrator/tests/test_heartbeat.py index 16a1281929..cd2b85f13b 100644 --- a/orchestrator/tests/test_heartbeat.py +++ b/orchestrator/tests/test_heartbeat.py @@ -2,22 +2,18 @@ Covers: -* ``should_fan_out_gateway_session`` (issue #2076 NB4) — the route- - level integration tests in ``test_messages.py`` exercise the - throttle through the HEARTBEAT endpoint, but a refactor of the - coordinator is hard to do safely without targeted unit coverage of - the disable case, the cooldown elapsed/not-elapsed branches, the - ``clear()`` reset, and concurrent access. * Per-slice independence of ``is_duplicate`` and ``check_rate_limit`` (issue #2471) — two slices that share a role must not share dedup state or rate budgets. + +The gateway-session fan-out throttle (``should_fan_out_gateway_session``) +and the orchestrator-mode guard were removed in #3164 along with the +in-pod wait arm; the coordinator no longer has any gateway side effect. """ from __future__ import annotations import sys -import threading -import time from pathlib import Path # Add orchestrator to path so bare imports resolve. @@ -35,172 +31,6 @@ # in ``conftest.py`` (autouse-scoped to all orchestrator tests). -class TestShouldFanOutGatewaySession: - """Unit tests for the per-(pipeline, slice, role) gateway-fan-out throttle.""" - - def test_zero_interval_disables_throttle(self): - """``min_interval_seconds == 0`` always returns True without recording.""" - coord = HeartbeatCoordinator() - - for _ in range(5): - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.0) is True - - # Nothing was recorded, so a subsequent positive-interval call - # sees no prior fan-out and fires. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - - def test_negative_interval_disables_throttle(self): - """Any non-positive value disables (matches the docstring contract).""" - coord = HeartbeatCoordinator() - - for _ in range(3): - assert coord.should_fan_out_gateway_session("p1", None, "coder", -1.0) is True - - # Same as the zero case — no recording happened. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - - def test_first_call_fires_and_records(self): - """First call fires AND records the new timestamp. - - The recording side-effect is verified directly: a second call - inside the cooldown window can only return False if the first - call wrote ``_last_fan_out[(p1, None, coder)]``. - """ - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - # Recording side-effect: second call within the window is suppressed. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - - def test_second_call_within_cooldown_suppressed(self): - """A second call inside the cooldown window returns False.""" - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - # Immediately again — well inside 30s. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - - def test_call_after_cooldown_fires(self): - """Once the window elapses, the next call fires again.""" - coord = HeartbeatCoordinator() - # 50ms cooldown + 200ms sleep — 150ms headroom keeps the test - # robust against GC pauses or noisy CI scheduling. ``time.sleep`` - # is a guaranteed lower bound, so the cooldown is always elapsed - # by the time the third assertion runs. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is True - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is False - time.sleep(0.2) - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is True - - def test_different_roles_throttled_independently(self): - """Throttle key includes the role — different roles don't collide.""" - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", None, "tester", 30.0) is True - # Both are now suppressed independently. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - assert coord.should_fan_out_gateway_session("p1", None, "tester", 30.0) is False - - def test_different_pipelines_throttled_independently(self): - """Throttle key includes the pipeline — different pipelines don't collide.""" - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p2", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - assert coord.should_fan_out_gateway_session("p2", None, "coder", 30.0) is False - - def test_different_slices_throttled_independently(self): - """Throttle key includes the slice — sibling slices don't collide (#2471).""" - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", "slice-2", "reviewer_code", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", "slice-3", "reviewer_code", 30.0) is True - # Each sibling is suppressed inside its own window — they did not - # share the throttle entry. - assert coord.should_fan_out_gateway_session("p1", "slice-2", "reviewer_code", 30.0) is False - assert coord.should_fan_out_gateway_session("p1", "slice-3", "reviewer_code", 30.0) is False - - def test_pipeline_level_and_sliced_dont_collide(self): - """``slice_id=None`` is a distinct bucket from any ``slice-`` (#2471).""" - coord = HeartbeatCoordinator() - # Pipeline-level agent fires. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - # Slice-scoped agent in the same pipeline + role still fires — - # it lives in a separate bucket. - assert coord.should_fan_out_gateway_session("p1", "slice-1", "coder", 30.0) is True - # Both are now suppressed inside their own windows. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - assert coord.should_fan_out_gateway_session("p1", "slice-1", "coder", 30.0) is False - - def test_suppressed_call_does_not_advance_recorded_timestamp(self): - """Hot-looping must not push the cooldown forward indefinitely. - - Uses a 50ms cooldown + 200ms sleep — 150ms headroom keeps the - ``time.time()`` reads robust against scheduling jitter on CI. - """ - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is True - # Hammer the coordinator inside the window — every call returns - # False, but the recorded timestamp stays at the initial fire. - for _ in range(10): - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is False - time.sleep(0.2) - # Still fires once the original window elapses, which proves the - # suppressed calls didn't reset the clock. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 0.05) is True - - def test_clear_drops_throttle_state_for_pipeline(self): - """``clear(pipeline)`` lets the next call fire immediately.""" - coord = HeartbeatCoordinator() - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - coord.clear("p1") - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - - def test_clear_only_affects_targeted_pipeline(self): - """``clear(p1)`` must not drop p2's throttle entries.""" - coord = HeartbeatCoordinator() - coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) - coord.should_fan_out_gateway_session("p2", None, "coder", 30.0) - coord.clear("p1") - # p1 reset, fires again. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - # p2 untouched, still suppressed. - assert coord.should_fan_out_gateway_session("p2", None, "coder", 30.0) is False - - def test_clear_drops_all_slice_scoped_entries_for_pipeline(self): - """``clear(p1)`` must sweep every slice's entry, not just ``slice_id=None`` (#2471).""" - coord = HeartbeatCoordinator() - coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) - coord.should_fan_out_gateway_session("p1", "slice-1", "coder", 30.0) - coord.should_fan_out_gateway_session("p1", "slice-2", "coder", 30.0) - coord.clear("p1") - # All three buckets reset — first post-clear call in each fires. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", "slice-1", "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", "slice-2", "coder", 30.0) is True - - def test_concurrent_callers_only_one_wins(self): - """Under contention, exactly one thread should see True per window.""" - coord = HeartbeatCoordinator() - results: list[bool] = [] - results_lock = threading.Lock() - barrier = threading.Barrier(20) - - def worker(): - barrier.wait() - res = coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) - with results_lock: - results.append(res) - - threads = [threading.Thread(target=worker) for _ in range(20)] - for t in threads: - t.start() - for t in threads: - t.join() - - # Exactly one thread crossed the throttle boundary first. - assert sum(results) == 1 - assert len(results) == 20 - - class TestIsDuplicate: """Per-(pipeline, slice, role) dedup memory (#2471).""" @@ -333,23 +163,19 @@ def test_reset_replaces_instance(self): # --------------------------------------------------------------------------- -# Slice-5: heartbeat coordinator mode guard (#3064 TASK-5-2) +# Orchestrator-mode heartbeat semantics (#3064 / #3164) # --------------------------------------------------------------------------- -# These tests are written test-first: they pin the slice-5 contract that the -# coder's TASK-5-1 must satisfy. Per the plan (slice-5) the -# HeartbeatCoordinator gains: -# -# * A mode guard: ``should_accept`` (or a method that checks -# EGG_EVENT_LOOP_OWNER) — in orchestrator mode, absent senders between -# events trip nothing; the heartbeat fan-out to gateway session refresh -# happens at spawn (slice-4) instead. -# -# Pod-mode behavior is unchanged (existing tests pass unmodified). +# Under the orchestrator-owned event loop, agents are spawned per-event and +# exit; between events there is no sender, so a heartbeat gap is normal. The +# gateway-session fan-out and the coordinator's mode flag were removed in +# #3164 — the coordinator now only tracks dedup/rate state and raises no +# alert itself. The remaining test pins that the coordinator can be queried +# without error for a key that has never sent a heartbeat. # --------------------------------------------------------------------------- class TestModeGuard: - """HeartbeatCoordinator mode-guard tests (#3064 slice-5 TASK-5-2).""" + """Coordinator has no gateway/alert side effect (#3064 / #3164).""" def test_absent_sender_in_orchestrator_mode_does_not_trip(self): """In orchestrator mode, an absent sender's silence is normal. @@ -372,47 +198,3 @@ def test_absent_sender_in_orchestrator_mode_does_not_trip(self): # confirms that the coordinator can be queried without error for # keys that have never sent a heartbeat. assert coord.is_duplicate("p1", "slice-5", "coder", "WORKING", None) is False - - def test_orchestrator_mode_suppresses_fan_out(self): - """``set_orchestrator_mode(True)`` short-circuits the fan-out gate. - - In orchestrator mode the gateway-session refresh happens at spawn - (slice-4 worktree re-attach), not from heartbeat fan-out, so - ``should_fan_out_gateway_session`` must return ``False`` regardless - of cooldown state — the orchestrator-mode early-return in - ``heartbeat.py`` precedes the cooldown bookkeeping. - """ - coord = HeartbeatCoordinator() - coord.set_orchestrator_mode(True) - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - # Still suppressed on a second call — the gate never records a - # timestamp in orchestrator mode, so there is no cooldown to expire. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - - def test_pod_mode_fan_out_unchanged(self): - """Default (pod) mode retains the cooldown-gated fan-out behavior. - - Without ``set_orchestrator_mode``, the first call fans out and - records a timestamp; the second call within the cooldown is - suppressed. This confirms the mode guard leaves pod-mode behavior - untouched. - """ - coord = HeartbeatCoordinator() # default pod mode - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - # Within cooldown — suppressed, exactly as before the mode guard. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - - def test_orchestrator_mode_toggle_restores_pod_behavior(self): - """Disabling orchestrator mode restores the fan-out path. - - ``set_orchestrator_mode(False)`` must un-gate the fan-out so a - coordinator that was flipped into orchestrator mode (e.g. on a - misconfigured restart) recovers pod-mode fan-out once corrected. - """ - coord = HeartbeatCoordinator() - coord.set_orchestrator_mode(True) - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False - coord.set_orchestrator_mode(False) - # Back in pod mode the first call fans out and records again. - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is True - assert coord.should_fan_out_gateway_session("p1", None, "coder", 30.0) is False diff --git a/orchestrator/tests/test_infra_error_escalation.py b/orchestrator/tests/test_infra_error_escalation.py index f0a0460ac8..175038a72d 100644 --- a/orchestrator/tests/test_infra_error_escalation.py +++ b/orchestrator/tests/test_infra_error_escalation.py @@ -992,6 +992,9 @@ def test_heartbeat_timeout_still_works(self): config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + # #3164: tripwires are active-Job-scoped (orchestrator owns the loop); + # mark the role as having a live one-shot Job so its silence trips. + monitor.set_active_roles({AGENT_ID}) _emit_heartbeat(bus, agent_id=AGENT_ID) with patch("health_monitor.time") as mock_time: @@ -1006,6 +1009,10 @@ def test_container_exit_still_works(self): bus = _make_event_bus() monitor = _make_monitor(bus) + # #3164: the container-exit tripwire is active-Job-scoped — the exiting + # role must have a live one-shot Job for the exit to escalate. + monitor.set_active_roles({AGENT_ID}) + escalations: list[dict] = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -1056,6 +1063,8 @@ def test_progress_stall_still_works(self): config = _make_config(orchestrator_heartbeat_timeout_seconds=60) monitor = _make_monitor(bus, config) + # #3164: tripwires are active-Job-scoped; mark the role's Job live. + monitor.set_active_roles({AGENT_ID}) _emit_progress(bus, agent_id=AGENT_ID) with patch("health_monitor.time") as mock_time: diff --git a/orchestrator/tests/test_kubernetes_spawner.py b/orchestrator/tests/test_kubernetes_spawner.py index 4257c788b3..010428d1af 100644 --- a/orchestrator/tests/test_kubernetes_spawner.py +++ b/orchestrator/tests/test_kubernetes_spawner.py @@ -1928,8 +1928,9 @@ def test_filesystem_scan_picks_up_slice_scoped_worktrees( # Test-first contract for TASK-2-2's one-shot spawn entry. The orchestrator # event loop (TASK-2-1) drives per-event Jobs through a NEW spawner entry # pinned here as ``KubernetesSpawner.spawn_event_job(...)``. It must: -# * set EGG_EVENT_LOOP_OWNER=orchestrator plus the event identity -# (EGG_EVENT_ACTION, EGG_EVENT_DEDUPE_KEY, payload refs) in Job env; +# * set the event identity (EGG_EVENT_ACTION, EGG_EVENT_DEDUPE_KEY, +# payload refs) in Job env (the EGG_EVENT_LOOP_OWNER flag was retired +# in #3164 — the wrapper keys on EGG_EVENT_ACTION alone); # * carry the dedupe key as a Job label (the reconciliation handle the # loop uses for stateless restart); # * derive the Job name from the existing @@ -1962,8 +1963,10 @@ def test_event_job_sets_owner_and_event_identity_env( repos=["owner/repo"], ) env = result.environment - # Owner flag flips the wrapper into its one-shot arm. - assert env["EGG_EVENT_LOOP_OWNER"] == "orchestrator" + # #3164 retired the EGG_EVENT_LOOP_OWNER ownership flag — the wrapper's + # one-shot handler keys on EGG_EVENT_ACTION alone, so the event job no + # longer injects an owner flag. + assert "EGG_EVENT_LOOP_OWNER" not in env # Full event identity is injected. assert env["EGG_EVENT_ACTION"] == "propose" assert env["EGG_EVENT_DEDUPE_KEY"] == self._KEY diff --git a/orchestrator/tests/test_messages.py b/orchestrator/tests/test_messages.py index 34243ae1a8..9d487f0d22 100644 --- a/orchestrator/tests/test_messages.py +++ b/orchestrator/tests/test_messages.py @@ -6,7 +6,6 @@ import json import sys -import time from pathlib import Path from unittest.mock import MagicMock, patch @@ -2750,238 +2749,6 @@ def test_heartbeat_route_accepts_optional_since_field(self, client, app): data = json.loads(resp.data) assert data["data"]["message"]["metadata"]["since"] == ("2026-04-23T07:00:00Z") - def test_heartbeat_refreshes_gateway_session(self, client, app): - """A real heartbeat fans out to the gateway to refresh the agent's session. - - Regression guard for #2068: without this, an agent in - ``WAITING_FOR_EVENT`` for >60 min has its gateway session pruned - as idle even though it's actively heartbeating. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": "fanout-role-coder", - "state": "WAITING_FOR_EVENT", - }, - ) - assert resp.status_code == 200 - mock_gw_client.heartbeat_session_by_container.assert_called_once_with( - "egg-agent-test-pipeline-fanout-role-coder" - ) - - def test_heartbeat_swallows_gateway_failure(self, client, app): - """Gateway fan-out failures must not fail the heartbeat call.""" - with app.test_request_context(): - mock_gw_client = MagicMock() - mock_gw_client.heartbeat_session_by_container.side_effect = RuntimeError("gateway down") - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": "fanout-role-tester", - "state": "WORKING", - }, - ) - assert resp.status_code == 200 - - def test_heartbeat_fan_out_fires_on_deduped_state(self, client, app): - """Deduped heartbeats still refresh the gateway session. - - Reviewer NB3 (#2068): the original fix only ran the fan-out - *after* dedup, so an agent stuck in ``WORKING`` through a long - compute (e.g. a slow ``make test``) would emit identical - heartbeats that were dropped before refreshing the session. The - fan-out now runs *between* dedup and rate-limit (see - ``routes/messages.py``): the dedup early-return invokes it so - unchanged-state heartbeats still count as gateway-session - liveness, while rate-limited heartbeats do not amplify into the - gateway. - - The ``_GATEWAY_FANOUT_MIN_INTERVAL_SECONDS`` cooldown (#2076 NB2) - is patched to ``0`` here so the test stays focused on the - dedup-fan-out invariant; the throttle's caps are pinned by - ``test_heartbeat_fan_out_throttle_*`` below. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - patch( - "routes.messages._GATEWAY_FANOUT_MIN_INTERVAL_SECONDS", - 0.0, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - # First WORKING heartbeat — recorded, not deduped. - resp1 = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": "fanout-dedup-role", - "state": "WORKING", - }, - ) - assert resp1.status_code == 200 - # Second identical heartbeat — dedup gate should drop it. - resp2 = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": "fanout-dedup-role", - "state": "WORKING", - }, - ) - assert resp2.status_code == 200 - assert json.loads(resp2.data)["data"]["deduped"] is True - # Both calls fan out — that's the bug fix. - assert mock_gw_client.heartbeat_session_by_container.call_count == 2 - - @pytest.mark.parametrize( - "from_role,expected_container_id", - [ - ("reviewer_refine", "egg-agent-test-pipeline-reviewer-refine"), - ("reviewer_code", "egg-agent-test-pipeline-reviewer-code"), - ("reviewer_agent_design", "egg-agent-test-pipeline-reviewer-agent-design"), - ("task_planner", "egg-agent-test-pipeline-task-planner"), - ("conflict_resolver", "egg-agent-test-pipeline-conflict-resolver"), - ("coder", "egg-agent-test-pipeline-coder"), - ], - ) - def test_heartbeat_fan_out_normalizes_underscores_to_hyphens( - self, client, app, from_role, expected_container_id - ): - """Fan-out container_id must mirror kubernetes_spawner's role hyphenation. - - Reviewer blocker (#2068 follow-up): k8s names are RFC-1123 - labels (no underscores), so ``kubernetes_spawner.JOB_NAME_FORMAT`` - is filled with ``role.replace("_", "-")``. ``from_role`` arrives - from ``EGG_AGENT_ROLE`` which is the underscore form, so the - fan-out must apply the same normalization — otherwise reviewer - roles like ``reviewer_refine`` build a container_id that never - matches the registered session and the gateway returns 404, - making the fan-out a silent no-op for exactly the BRC reviewer - roles that #2068 most acutely affects. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": from_role, - "state": "WAITING_FOR_EVENT", - }, - ) - assert resp.status_code == 200 - mock_gw_client.heartbeat_session_by_container.assert_called_once_with( - expected_container_id - ) - - @pytest.mark.parametrize( - "from_role,slice_id,expected_container_id", - [ - ( - "reviewer_contract", - "slice-2", - "egg-agent-test-pipeline-slice-2-reviewer-contract", - ), - ( - "reviewer_code_holistic", - "slice-12", - "egg-agent-test-pipeline-slice-12-reviewer-code-holistic", - ), - ("tester", "slice-5", "egg-agent-test-pipeline-slice-5-tester"), - ("coder", "slice-1", "egg-agent-test-pipeline-slice-1-coder"), - ], - ) - def test_heartbeat_fan_out_includes_slice_id( - self, client, app, from_role, slice_id, expected_container_id - ): - """Slice-scoped heartbeats fan out to the slice-scoped container_id (#2451). - - Slice-DAG agents register gateway sessions under - ``egg-agent-{pid}-{slice_id}-{role}`` per - ``kubernetes_spawner.JOB_NAME_FORMAT_SLICE``. The orchestrator - must thread the agent's ``slice_id`` (forwarded via the - heartbeat body from ``EGG_SLICE_ID``) into the fan-out's - container_id, otherwise every slice-scoped reviewer/tester - emits warnings ("Session not found for container") and the - gateway session never has its idle timer refreshed. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": from_role, - "state": "WAITING_FOR_EVENT", - "slice_id": slice_id, - }, - ) - assert resp.status_code == 200 - mock_gw_client.heartbeat_session_by_container.assert_called_once_with( - expected_container_id - ) - @pytest.mark.parametrize( "bad_slice_id", [ @@ -3007,16 +2774,9 @@ def test_heartbeat_rejects_invalid_slice_id(self, client, app, bad_slice_id): security intent of the regex is explicit in the test corpus. """ with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): + with patch( + "routes.messages.get_state_store_for_pipeline" + ) as mock_get_store_for_pipeline: mock_get_store_for_pipeline.return_value = ( MagicMock(), _make_pipeline_mock(), @@ -3032,14 +2792,13 @@ def test_heartbeat_rejects_invalid_slice_id(self, client, app, bad_slice_id): # Empty-string ``slice_id`` is treated as "no slice" by # the orchestrator's ``extract_slice_id`` (matches the # agent-side ``or`` fallback) — the request still - # succeeds but pipeline-level fan-out semantics apply. + # succeeds but pipeline-level semantics apply. if bad_slice_id == "": assert resp.status_code == 200 return assert resp.status_code == 400, ( f"slice_id={bad_slice_id!r} must reject with 400; got {resp.status_code}" ) - mock_gw_client.heartbeat_session_by_container.assert_not_called() def test_heartbeat_message_metadata_carries_slice_id(self, client, app): """Slice-scoped HEARTBEATs land on the bus with ``slice_id`` in @@ -3055,16 +2814,9 @@ def test_heartbeat_message_metadata_carries_slice_id(self, client, app): store = get_message_store() with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): + with patch( + "routes.messages.get_state_store_for_pipeline" + ) as mock_get_store_for_pipeline: mock_get_store_for_pipeline.return_value = ( MagicMock(), _make_pipeline_mock(), @@ -3098,16 +2850,9 @@ def test_heartbeat_metadata_omits_slice_id_when_pipeline_level(self, client, app store = get_message_store() with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - ): + with patch( + "routes.messages.get_state_store_for_pipeline" + ) as mock_get_store_for_pipeline: mock_get_store_for_pipeline.return_value = ( MagicMock(), _make_pipeline_mock(), @@ -3125,160 +2870,6 @@ def test_heartbeat_metadata_omits_slice_id_when_pipeline_level(self, client, app f"Pipeline-level HEARTBEAT must omit slice_id key, got: {heartbeats[-1].metadata}" ) - def test_heartbeat_sibling_slices_do_not_share_throttle(self, client, app): - """Sibling slices with the same role each fan out independently (#2451). - - The dedup-amplification cap (#2076 NB2) is keyed per role to - bound a single hot-looping agent. Without slice-scoping the - throttle key, slice-2 reviewer-code's fan-out would suppress - slice-3 reviewer-code's fan-out for 30 s, leaving slice-3's - gateway session unrefreshed even though they are independent - agents in independent pods. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - patch( - "routes.messages._GATEWAY_FANOUT_MIN_INTERVAL_SECONDS", - 300.0, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - for slice_id in ("slice-2", "slice-3"): - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={ - "from_role": "reviewer_code", - "state": "WORKING", - "slice_id": slice_id, - }, - ) - assert resp.status_code == 200 - assert mock_gw_client.heartbeat_session_by_container.call_count == 2 - fan_out_calls = [ - call.args[0] - for call in mock_gw_client.heartbeat_session_by_container.call_args_list - ] - assert fan_out_calls == [ - "egg-agent-test-pipeline-slice-2-reviewer-code", - "egg-agent-test-pipeline-slice-3-reviewer-code", - ] - - def test_heartbeat_fan_out_throttle_caps_dedup_amplification(self, client, app): - """#2076 NB2: dedup'd hot-loops cannot amplify into the gateway. - - The dedup early-return path bypasses the per-role rate limit by - design (#1897 NB1: dedup'd heartbeats are no-ops and must not - consume rate budget). Without a separate cap, a misbehaving - agent hot-looping with identical state could fan out a gateway - session refresh on every call. ``_refresh_gateway_session`` - applies a per-role cooldown via - ``HeartbeatCoordinator.should_fan_out_gateway_session`` to bound - amplification. - - Five back-to-back identical heartbeats (1 fresh + 4 dedup'd) - within the cooldown window MUST produce exactly one fan-out. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - # Cooldown well above realistic test wall-clock so all - # five posts fall inside the same window. - patch( - "routes.messages._GATEWAY_FANOUT_MIN_INTERVAL_SECONDS", - 300.0, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - role = "fanout-throttle-hotloop-role" - for _ in range(5): - resp = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={"from_role": role, "state": "WORKING"}, - ) - assert resp.status_code == 200 - assert mock_gw_client.heartbeat_session_by_container.call_count == 1 - - def test_heartbeat_fan_out_throttle_resumes_after_window(self, client, app): - """#2076 NB2: cooldown is a throttle, not a one-shot mute. - - After the per-role cooldown elapses, the next heartbeat MUST - fan out again — otherwise a long-running agent in a single - state would fan out exactly once and then silently age out of - the gateway's 60-minute idle window. - - Uses a tiny cooldown + ``time.sleep`` rather than mocking - ``time.time`` so the patch doesn't ripple through unrelated - callers (Flask internals, gateway client) inside the request - scope. Uses ``WAITING_FOR_EVENT`` (dedup-exempt) so this - exercise also pins the throttle on the post-rate-limit fan-out - site, not just the dedup early-return. - """ - with app.test_request_context(): - mock_gw_client = MagicMock() - with ( - patch( - "routes.messages.get_state_store_for_pipeline" - ) as mock_get_store_for_pipeline, - patch( - "gateway_client.get_gateway_client", - return_value=mock_gw_client, - ), - patch( - "routes.messages._GATEWAY_FANOUT_MIN_INTERVAL_SECONDS", - 0.05, - ), - ): - mock_get_store_for_pipeline.return_value = ( - MagicMock(), - _make_pipeline_mock(), - ) - role = "fanout-throttle-window-role" - # First heartbeat — fans out. - resp1 = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={"from_role": role, "state": "WAITING_FOR_EVENT"}, - ) - assert resp1.status_code == 200 - assert mock_gw_client.heartbeat_session_by_container.call_count == 1 - - # Inside the 50 ms cooldown — no additional fan-out. - resp2 = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={"from_role": role, "state": "WAITING_FOR_EVENT"}, - ) - assert resp2.status_code == 200 - assert mock_gw_client.heartbeat_session_by_container.call_count == 1 - - # Past the cooldown — fan-out fires again. - time.sleep(0.07) - resp3 = client.post( - "/api/v1/pipelines/test-pipeline/heartbeat", - json={"from_role": role, "state": "WAITING_FOR_EVENT"}, - ) - assert resp3.status_code == 200 - assert mock_gw_client.heartbeat_session_by_container.call_count == 2 - class TestWaitTimeoutFloorRegression: """Plan non-blocking: ``timeout <= 0`` is silently coerced to 1s diff --git a/orchestrator/tests/test_models.py b/orchestrator/tests/test_models.py index 519d849535..3ef64c182c 100644 --- a/orchestrator/tests/test_models.py +++ b/orchestrator/tests/test_models.py @@ -569,11 +569,17 @@ def test_max_parallel_slices_rejects_below_one(self): with pytest.raises(ValidationError): PipelineConfig(max_parallel_slices=0) - def test_env_fallback_default_is_single_slice(self): - """When no per-pipeline value is set, the fallback default is 1 slice.""" + def test_env_fallback_default_is_four_slices(self): + """When no per-pipeline value is set, the fallback default is 4 slices. + + Raised from 1 to 4 in #3164: under the orchestrator-owned event + loop an in-flight slice no longer pins a resident ~8-container + cohort (it spawns one short-lived one-shot pod per BRC event), so + a 4-wide wave is safe on a single-node host. + """ from orchestrator.env_config import DEFAULT_MAX_PARALLEL_SLICES - assert DEFAULT_MAX_PARALLEL_SLICES == 1 + assert DEFAULT_MAX_PARALLEL_SLICES == 4 class TestResolveConsensusTimeoutMinutes: diff --git a/orchestrator/tests/test_next_action_iteration_feedback.py b/orchestrator/tests/test_next_action_iteration_feedback.py index d34d9a779e..0ca7ecfcb2 100644 --- a/orchestrator/tests/test_next_action_iteration_feedback.py +++ b/orchestrator/tests/test_next_action_iteration_feedback.py @@ -1,8 +1,8 @@ """Tests for the per-iteration operator kickback delivery on the orchestrator-owned event-loop ``next-action`` path (#3231). -Under ``EGG_EVENT_LOOP_OWNER=orchestrator`` the re-spawned producer's -prompt is composed from the ``event_payload`` the ``next-action`` route +The re-spawned producer's prompt is composed from the +``event_payload`` the ``next-action`` route returns. Without the ``iteration_feedback`` block the producer re-reads its own prior on-disk draft and re-proposes it byte-for-byte — the operator's ``request_changes`` / ``change_approach`` silently no-ops diff --git a/orchestrator/tests/test_overseer_hitl_integration.py b/orchestrator/tests/test_overseer_hitl_integration.py index 4107ed848f..a994ffae6a 100644 --- a/orchestrator/tests/test_overseer_hitl_integration.py +++ b/orchestrator/tests/test_overseer_hitl_integration.py @@ -105,6 +105,7 @@ def test_stall_escalates_immediately_to_overseer(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -146,6 +147,7 @@ def test_overseer_disabled_escalates_directly_to_hitl(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -280,6 +282,7 @@ def test_container_exit_bypasses_overseer(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) diff --git a/orchestrator/tests/test_pipeline_failure_path.py b/orchestrator/tests/test_pipeline_failure_path.py index af4c55aec9..f36bd829d9 100644 --- a/orchestrator/tests/test_pipeline_failure_path.py +++ b/orchestrator/tests/test_pipeline_failure_path.py @@ -13,6 +13,8 @@ from pathlib import Path from unittest.mock import MagicMock, call, patch +import pytest + _orchestrator_path = Path(__file__).parent.parent if str(_orchestrator_path) not in sys.path: sys.path.insert(0, str(_orchestrator_path)) @@ -31,6 +33,28 @@ from models import Pipeline, PipelinePhase, PipelineStatus +@pytest.fixture(autouse=True) +def _stub_concurrent_phase(): + """Stub the concurrent-phase poll loop to a fast failure outcome. + + #3164: ``_run_pipeline``'s concurrent phases (refine/plan/implement) run + through the real ``_run_concurrent_phase``. Now that the orchestrator + unconditionally owns the BRC event loop, ``spawn_all`` returns ``[]`` and + the poll loop waits the full consensus timeout (≥60s floor) when agents + are mocked and never confirm — pod mode used to fast-fail on the + empty-container path. These tests exercise ``_run_pipeline``'s + orchestration (failure handling, worktree push/cleanup, network-mode + detection, base-branch forwarding, finally-cleanup), not the + concurrent-phase internals, so stub the phase to return immediately with + the failure outcome the suite's ``_spawn_and_wait`` mock intended. The + four ``TestZombiePipelineSyntheticEvent`` tests that patch + ``_run_concurrent_phase`` with their own side effect override this within + their scope. + """ + with patch("routes.pipelines._run_concurrent_phase", return_value=(1, "error log")): + yield + + def _make_running_pipeline(branch="egg/issue-42"): """Create a Pipeline in RUNNING state with its current phase also RUNNING.""" pipeline = Pipeline( diff --git a/orchestrator/tests/test_prompt_sync_ratchet.py b/orchestrator/tests/test_prompt_sync_ratchet.py index 63dad69474..700411998d 100644 --- a/orchestrator/tests/test_prompt_sync_ratchet.py +++ b/orchestrator/tests/test_prompt_sync_ratchet.py @@ -126,7 +126,7 @@ # prompt-constructing string — agents never see it. Reworking the # comment to drop the literal token would strip the pointer # reviewers depend on to navigate the deletion. - ("event_prompt.py", "git fetch", 1254), + ("event_prompt.py", "git fetch", 1253), ) diff --git a/orchestrator/tests/test_restart_agent.py b/orchestrator/tests/test_restart_agent.py index de9bbd3e02..f8cf5cc86d 100644 --- a/orchestrator/tests/test_restart_agent.py +++ b/orchestrator/tests/test_restart_agent.py @@ -406,7 +406,13 @@ class TestRestartAgentEndpoint: def test_restart_agent_success( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Successful agent restart returns 200 with new container ID.""" + """Successful agent restart returns 200 and delegates respawn (#3164). + + After #3164 ``restart_agent`` no longer spawns a resident pod — + it resets consensus + kills the live one-shot Job and the event + loop respawns. So the response carries no ``container_id``; it + carries the ``respawn`` delegation marker. + """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -416,18 +422,7 @@ def test_restart_agent_success( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -439,8 +434,11 @@ def test_restart_agent_success( assert response.status_code == 200 data = response.get_json() assert data.get("success") is True - assert data["data"]["container_id"] == "new-container-xyz" + # #3164: no resident pod is spawned here. + mock_spawner.restart_agent_container.assert_not_called() + assert "container_id" not in data["data"] assert data["data"]["agent_role"] == "coder" + assert data["data"]["respawn"] == "delegated to orchestrator event loop" @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") @@ -517,18 +515,7 @@ def test_restart_agent_cancelled_pipeline_resumes( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -562,18 +549,7 @@ def test_restart_agent_failed_pipeline( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -590,10 +566,15 @@ def test_restart_agent_failed_pipeline( @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_restart_spawner_failure_returns_500( + def test_restart_tolerates_live_job_listing_failure( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Restart returns 500 when spawner fails.""" + """A k8s failure listing live one-shot Jobs is best-effort (#3164). + + The respawn is owned by the event loop, so a failure tearing + down the stuck pod must not fail the restart — consensus is + still reset and the route returns 200. + """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -603,7 +584,8 @@ def test_restart_spawner_failure_returns_500( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - mock_spawner.restart_agent_container.side_effect = ContainerSpawnError("Failed") + mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") + mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -611,21 +593,23 @@ def test_restart_spawner_failure_returns_500( json={}, ) - assert response.status_code == 500 + assert response.status_code == 200 + mock_spawner.restart_agent_container.assert_not_called() @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_restart_spawner_failure_reverts_status_to_failed( + def test_restart_failed_pipeline_transitions_to_running_no_revert( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Spawn failure must revert pipeline status back to FAILED. + """Restart of a FAILED pipeline transitions to RUNNING and stays there. - Regression test for review feedback on #1594: the early status update - sets RUNNING before the spawn attempt. If the spawn raises - ContainerSpawnError, the status must be reverted so monitoring doesn't - see a 'running' pipeline with no running agent. + After #3164 there is no resident spawn that can fail, so the + early FAILED -> RUNNING transition is never reverted: the event + loop owns the respawn. Replaces the old + ``...spawner_failure_reverts_status_to_failed`` regression test + whose spawn-failure path no longer exists. """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -638,7 +622,8 @@ def test_restart_spawner_failure_reverts_status_to_failed( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - mock_spawner.restart_agent_container.side_effect = ContainerSpawnError("Failed") + mock_spawner.k8s.list_containers.return_value = [] + mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -646,27 +631,23 @@ def test_restart_spawner_failure_reverts_status_to_failed( json={}, ) - assert response.status_code == 500 - # Verify status was reverted to FAILED after spawn failure - assert pipeline.status == PipelineStatus.FAILED - assert pipeline.phases["implement"].status == PipelineStatus.FAILED - # Verify update_pipeline was called at least twice: - # once for the early RUNNING update, once for the FAILED revert - assert mock_store.update_pipeline.call_count >= 2 + assert response.status_code == 200 + assert pipeline.status == PipelineStatus.RUNNING + assert pipeline.phases["implement"].status == PipelineStatus.RUNNING + mock_spawner.restart_agent_container.assert_not_called() @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_restart_cancelled_spawner_failure_reverts_status_to_failed( + def test_restart_cancelled_pipeline_transitions_to_running_no_revert( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Spawn failure on a CANCELLED pipeline reverts status to FAILED. + """Restart of a CANCELLED pipeline transitions to RUNNING (no revert). - When restart_agent is called on a CANCELLED pipeline and the spawn - fails, the revert unconditionally sets FAILED (not back to CANCELLED). - This is intentional: FAILED is also restartable, and it accurately - reflects that the restart attempt failed. + cancel_task(cleanup=false) leaves CANCELLED with state preserved; + restart resumes it. With no resident spawn (#3164) there is no + failure path that would revert to FAILED. """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -679,7 +660,8 @@ def test_restart_cancelled_spawner_failure_reverts_status_to_failed( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - mock_spawner.restart_agent_container.side_effect = ContainerSpawnError("Failed") + mock_spawner.k8s.list_containers.return_value = [] + mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -687,11 +669,10 @@ def test_restart_cancelled_spawner_failure_reverts_status_to_failed( json={}, ) - assert response.status_code == 500 - # Spawn failure reverts to FAILED (not back to CANCELLED) - assert pipeline.status == PipelineStatus.FAILED - assert pipeline.phases["implement"].status == PipelineStatus.FAILED - assert mock_store.update_pipeline.call_count >= 2 + assert response.status_code == 200 + assert pipeline.status == PipelineStatus.RUNNING + assert pipeline.phases["implement"].status == PipelineStatus.RUNNING + mock_spawner.restart_agent_container.assert_not_called() @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") @@ -709,254 +690,6 @@ def test_restart_invalid_pipeline_id_format(self, mock_repo, mock_resolve, clien assert response.status_code == 400 - @patch("routes.pipelines._compute_gateway_mode") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_restart_agent_uses_computed_gateway_mode( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_gateway_mode, client - ): - """Agent restart should use _compute_gateway_mode, not hardcoded 'public'. - - This ensures private-repo pipelines get the correct gateway mode on restart. - """ - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - mock_gateway_mode.return_value = ("private", "private") - - pipeline = _make_pipeline_with_running_agent() - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={}, - ) - - assert response.status_code == 200 - # Verify _compute_gateway_mode was called with the pipeline - mock_gateway_mode.assert_called_once_with(pipeline) - # Verify the computed mode was passed to the spawner - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call[1].get("mode") == "private", ( - "Expected computed gateway mode 'private', not hardcoded 'public'" - ) - - -# --------------------------------------------------------------------------- -# Issue #2967 follow-up: restart path resolves ``pipeline.base_branch=None`` -# --------------------------------------------------------------------------- - - -@pytest.mark.skipif(not _HAS_FLASK, reason="Flask not available") -class TestRestartAgentBaseBranchResolution: - """The restart path resolves the default branch when - ``pipeline.base_branch`` is ``None`` so the spawner receives a concrete - branch name. Without this, the spawner's ``EGG_BASE_BRANCH`` - env-injection guard skips the export, the BRC event-pump's per-producer - ``git log {sha}..HEAD --not origin/`` delta falls back to - ``origin/main`` in both consumers, and the diff errors out on every - non-``main`` repo (mirrors the initial-spawn fix at - ``test_kubernetes_spawner.py::test_spawn_with_base_branch_sets_egg_base_branch_env``). - """ - - @patch("routes.pipelines.get_default_branch") - @patch("routes.pipelines.resolve_worktree_repo_path") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_restart_resolves_base_branch_when_pipeline_base_is_none( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_resolve_worktree, - mock_get_default, - client, - ): - """A ``None`` ``pipeline.base_branch`` resolves to ``get_default_branch`` - and is forwarded as the spawner's ``base_branch`` argument.""" - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - mock_resolve_worktree.return_value = Path("/repo/worktree") - mock_get_default.return_value = "master" - - pipeline = _make_pipeline_with_running_agent() - assert pipeline.base_branch is None, ( - "fixture must leave base_branch unset to exercise the auto-detect path" - ) - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={"reason": "auto-detect base branch"}, - ) - - assert response.status_code == 200 - restart_kwargs = mock_spawner.restart_agent_container.call_args.kwargs - assert restart_kwargs.get("base_branch") == "master", ( - "Restart must thread the resolved default branch into the spawner's " - "base_branch kwarg so EGG_BASE_BRANCH is exported and the BRC delta " - "works on non-main repos (#2967 follow-up)." - ) - - @patch("routes.pipelines.get_default_branch") - @patch("routes.pipelines.resolve_worktree_repo_path") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_restart_preserves_explicit_pipeline_base_branch( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_resolve_worktree, - mock_get_default, - client, - ): - """An explicit ``pipeline.base_branch`` is forwarded unchanged — the - ``get_default_branch`` fallback only fires when it's ``None``.""" - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - mock_resolve_worktree.return_value = Path("/repo/worktree") - # If this fires, the explicit ``base_branch`` was wrongly discarded. - mock_get_default.side_effect = AssertionError( - "get_default_branch must not fire when pipeline.base_branch is set" - ) - - pipeline = _make_pipeline_with_running_agent() - pipeline.base_branch = "jwbron-claude-md" - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={"reason": "explicit base branch"}, - ) - - assert response.status_code == 200 - restart_kwargs = mock_spawner.restart_agent_container.call_args.kwargs - assert restart_kwargs.get("base_branch") == "jwbron-claude-md" - - @patch("routes.pipelines.get_default_branch") - @patch("routes.pipelines.resolve_worktree_repo_path") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_restart_tolerates_worktree_resolution_failure( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_resolve_worktree, - mock_get_default, - client, - ): - """If the worktree path cannot be resolved (e.g. pruned), the restart - still proceeds with the raw ``pipeline.base_branch`` — preserving - pre-#2967 restart availability rather than blocking on resolution.""" - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - mock_resolve_worktree.side_effect = RuntimeError("worktree pruned") - - pipeline = _make_pipeline_with_running_agent() - # Leave base_branch=None so the resolution would normally fire. - assert pipeline.base_branch is None - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={"reason": "worktree pruned"}, - ) - - assert response.status_code == 200 - # get_default_branch must not be called when worktree resolution fails. - mock_get_default.assert_not_called() - # Restart still happens with the raw (None) pipeline.base_branch. - restart_kwargs = mock_spawner.restart_agent_container.call_args.kwargs - assert restart_kwargs.get("base_branch") is None - # --------------------------------------------------------------------------- # Issue #2410: slice_id forwarding through the operator restart route @@ -971,10 +704,17 @@ class TestRestartAgentEndpointSliceScope: @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_slice_id_query_param_forwarded_to_spawner( + def test_slice_id_query_param_scopes_live_job_lookup_and_count( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """``?slice_id=slice-2`` reaches ``restart_agent_container``.""" + """``?slice_id=slice-2`` scopes the live-Job label filter + restart count (#2410/#3164). + + After #3164 there is no spawn; the slice scope instead narrows + the ``list_containers`` label filter (so we only tear down the + slice's stuck pod) and the per-slice restart-count bucket. + """ + from kubernetes_client import LABEL_AGENT_ROLE, LABEL_PIPELINE_ID, LABEL_SLICE_ID + mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -984,18 +724,7 @@ def test_slice_id_query_param_forwarded_to_spawner( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -1005,24 +734,31 @@ def test_slice_id_query_param_forwarded_to_spawner( ) assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["slice_id"] == "slice-2" + mock_spawner.restart_agent_container.assert_not_called() + # The live one-shot Job lookup is found by LABEL (Jobs carry an + # event-discriminator suffix in their name), scoped to the slice. + list_call = mock_spawner.k8s.list_containers.call_args + assert list_call.kwargs["labels"] == { + LABEL_PIPELINE_ID: "issue-100", + LABEL_AGENT_ROLE: "coder", + LABEL_SLICE_ID: "slice-2", + } # Reading the restart count after a slice-scoped restart MUST - # query the per-slice budget bucket (#2410). Without this, the - # JSON response and audit log report the pipeline-level count - # (typically 0) and operators can't trust "you've burned N of M - # restarts" telemetry. + # query the per-slice budget bucket (#2410). get_count_call = mock_spawner.get_restart_count.call_args assert get_count_call.kwargs.get("slice_id") == "slice-2" + assert response.get_json()["data"]["slice_id"] == "slice-2" @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_slice_id_body_field_forwarded_to_spawner( + def test_slice_id_body_field_scopes_live_job_lookup_and_count( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """``{"slice_id": "slice-2"}`` in the body reaches the spawner.""" + """``{"slice_id": "slice-2"}`` in the body scopes the label filter + count.""" + from kubernetes_client import LABEL_AGENT_ROLE, LABEL_PIPELINE_ID, LABEL_SLICE_ID + mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -1032,18 +768,7 @@ def test_slice_id_body_field_forwarded_to_spawner( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -1053,8 +778,13 @@ def test_slice_id_body_field_forwarded_to_spawner( ) assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["slice_id"] == "slice-2" + mock_spawner.restart_agent_container.assert_not_called() + list_call = mock_spawner.k8s.list_containers.call_args + assert list_call.kwargs["labels"] == { + LABEL_PIPELINE_ID: "issue-100", + LABEL_AGENT_ROLE: "coder", + LABEL_SLICE_ID: "slice-2", + } get_count_call = mock_spawner.get_restart_count.call_args assert get_count_call.kwargs.get("slice_id") == "slice-2" @@ -1082,10 +812,12 @@ def test_invalid_slice_id_returns_400(self, mock_repo, mock_resolve, client): @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_no_slice_id_forwards_none( + def test_no_slice_id_omits_slice_label_and_uses_none_count( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Pipeline-level callers (no slice_id) get ``slice_id=None`` forwarded.""" + """Pipeline-level callers omit the slice label and read the None count bucket.""" + from kubernetes_client import LABEL_AGENT_ROLE, LABEL_PIPELINE_ID, LABEL_SLICE_ID + mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -1095,18 +827,7 @@ def test_no_slice_id_forwards_none( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -1116,8 +837,13 @@ def test_no_slice_id_forwards_none( ) assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["slice_id"] is None + mock_spawner.restart_agent_container.assert_not_called() + list_call = mock_spawner.k8s.list_containers.call_args + assert list_call.kwargs["labels"] == { + LABEL_PIPELINE_ID: "issue-100", + LABEL_AGENT_ROLE: "coder", + } + assert LABEL_SLICE_ID not in list_call.kwargs["labels"] get_count_call = mock_spawner.get_restart_count.call_args assert get_count_call.kwargs.get("slice_id") is None @@ -1201,18 +927,7 @@ def test_known_slice_id_passes_validation( ) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -1224,12 +939,12 @@ def test_known_slice_id_passes_validation( assert response.status_code == 200 # Lock in the "gate ran AND accepted" intent: without this assertion # the test would pass identically whether the gate executed or fell - # through silently on a fast-path skip. ``load_contract`` may also - # be invoked downstream by the spawner flow, so we verify the gate's - # specific call rather than the total count. + # through silently on a fast-path skip. mock_load_contract.assert_any_call(100, Path("/repo")) - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["slice_id"] == "slice-2" + # #3164: validation accepted, no resident spawn; the per-slice + # count bucket is read. + mock_spawner.restart_agent_container.assert_not_called() + assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-2" @patch("egg_contracts.loader.load_contract") @patch("routes.pipelines.get_container_spawner") @@ -1276,765 +991,223 @@ def test_slice_id_rejected_for_pipeline_without_contract( mock_load_contract.assert_not_called() mock_spawner.restart_agent_container.assert_not_called() - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_slice_restart_passes_slice_integration_branch( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client - ): - """Slice-scoped restart must target the slice integration branch (#2428). - - The gateway registers the new session with ``assigned_branch = - branch`` and rejects every push that doesn't match. If the - restart route forwards the pipeline tip instead, the restarted - agent's pushes are rejected the same way the slice-coder - spawn-side bug rejected them. The slice integration branch is - ``/``, the same shape the slice - scheduler uses for the initial spawn. - """ - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - # Use a /work-suffixed pipeline branch so the namespace root is - # exercised (post-#2399 shape). - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-2", - json={"reason": "Slice agent stalled"}, - ) - - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["branch"] == "egg/issue-100/slice-2" - - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_pipeline_level_restart_passes_pipeline_branch( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client - ): - """Without slice_id, restart still forwards the pipeline branch unchanged.""" - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner +# --------------------------------------------------------------------------- +# Issue #2759: omitted slice_id is derived from the phase's agent records +# --------------------------------------------------------------------------- - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={"reason": "Pipeline-level restart"}, - ) - - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["branch"] == "egg/issue-100" - - -# --------------------------------------------------------------------------- -# Issue #2759: omitted slice_id is derived from the phase's agent records -# --------------------------------------------------------------------------- - - -def _make_pipeline_with_slice_agents(role, slice_records, pipeline_id="issue-100"): - """Pipeline whose current phase carries per-slice ``AgentExecution`` records. - - ``slice_records`` is a list of ``(slice_id, status)`` tuples — one per - slice that has run (or is running) ``role``. Exercises the #2759 slice - auto-derivation in ``restart_agent``. - """ - pipeline = Pipeline( - id=pipeline_id, - issue_number=100, - repo="owner/repo", - branch="egg/issue-100/work", - status=PipelineStatus.RUNNING, - current_phase=PipelinePhase.IMPLEMENT, - ) - pipeline.phases = { - "implement": PhaseExecution( - phase=PipelinePhase.IMPLEMENT, - status=PipelineStatus.RUNNING, - agents=[ - AgentExecution( - role=role, - status=status, - container_id=f"container-{sid}", - slice_id=sid, - ) - for sid, status in slice_records - ], - ), - } - return pipeline - - -@pytest.mark.skipif(not _HAS_FLASK, reason="Flask not available") -class TestRestartAgentEndpointSliceDerivation: - """Omitted ``slice_id`` is derived from phase agent records (#2759). - - A slice-mode restart that omits ``slice_id`` would otherwise spawn the - agent pipeline-level — ``EGG_SLICE_ID`` unset — so its BRC signals - route to the bare pipeline tracker instead of the slice's tracker and - the slice's consensus wedges with no message-bus recovery path. - """ - - @patch("egg_contracts.loader.load_contract") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_omitted_slice_id_derived_from_single_non_complete_record( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_load_contract, client - ): - """One non-complete slice record → its slice_id is derived and forwarded.""" - from egg_contracts.models import Contract, Slice - - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - # slice-1 finished cleanly; slice-2's reviewer crashed. - pipeline = _make_pipeline_with_slice_agents( - AgentRole.REVIEWER_CODE_HOLISTIC, - [ - ("slice-1", AgentExecutionStatus.COMPLETE), - ("slice-2", AgentExecutionStatus.FAILED), - ], - ) - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_load_contract.return_value = Contract( - pipeline_id="issue-100", - slices=[Slice(id="slice-1", name="First"), Slice(id="slice-2", name="Second")], - ) - - mock_spawner = MagicMock() - mock_spawner.restart_agent_container.return_value = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-reviewer_code_holistic", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.REVIEWER_CODE_HOLISTIC, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", - json={"reason": "container crashed"}, - ) - - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - # The crashed slice is derived: the spawner sets EGG_SLICE_ID from - # this and targets the slice integration branch, so the respawned - # agent rejoins slice-2's BRC tracker rather than the bare one. - assert restart_call.kwargs["slice_id"] == "slice-2" - assert restart_call.kwargs["branch"] == "egg/issue-100/slice-2" - - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_omitted_slice_id_ambiguous_rejected( - self, mock_repo, mock_resolve, mock_spawner_fn, client - ): - """Multiple non-complete slice records → 400 with the candidate list.""" - mock_repo.return_value = "/repo" - pipeline = _make_pipeline_with_slice_agents( - AgentRole.REVIEWER_CODE_HOLISTIC, - [ - ("slice-1", AgentExecutionStatus.COMPLETE), - ("slice-2", AgentExecutionStatus.FAILED), - ("slice-3", AgentExecutionStatus.RUNNING), - ], - ) - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - mock_spawner = MagicMock() - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", - json={}, - ) - - assert response.status_code == 400 - body = response.get_json() - assert body["success"] is False - assert body["reason"] == "slice_id_required" - assert body["details"]["restart_candidates"] == ["slice-2", "slice-3"] - assert body["details"]["known_slices"] == ["slice-1", "slice-2", "slice-3"] - # Never silently spawn an unscoped agent on an ambiguous restart. - mock_spawner.restart_agent_container.assert_not_called() - - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_omitted_slice_id_all_complete_rejected( - self, mock_repo, mock_resolve, mock_spawner_fn, client - ): - """All slice records complete → 400 (no unambiguous restart target).""" - mock_repo.return_value = "/repo" - pipeline = _make_pipeline_with_slice_agents( - AgentRole.REVIEWER_CODE_HOLISTIC, - [ - ("slice-1", AgentExecutionStatus.COMPLETE), - ("slice-2", AgentExecutionStatus.COMPLETE), - ], - ) - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - mock_spawner = MagicMock() - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", - json={}, - ) - - assert response.status_code == 400 - body = response.get_json() - assert body["reason"] == "slice_id_required" - assert body["details"]["restart_candidates"] == [] - mock_spawner.restart_agent_container.assert_not_called() - - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_explicit_slice_id_bypasses_derivation( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client - ): - """An explicit slice_id wins even when records would derive otherwise. - - Derivation runs only when ``slice_id`` is absent; an explicit - value (even one with no agent record) is forwarded as-is, so this - path does not regress the #2410 explicit-scope contract. - """ - from egg_contracts.models import Contract, Slice - - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_slice_agents( - AgentRole.REVIEWER_CODE_HOLISTIC, - [("slice-2", AgentExecutionStatus.FAILED)], - ) - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - mock_spawner.restart_agent_container.return_value = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-1-reviewer_code_holistic", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.REVIEWER_CODE_HOLISTIC, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - with patch("egg_contracts.loader.load_contract") as mock_load_contract: - mock_load_contract.return_value = Contract( - pipeline_id="issue-100", - slices=[Slice(id="slice-1", name="First"), Slice(id="slice-2", name="Second")], - ) - response = client.post( - "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart?slice_id=slice-1", - json={}, - ) - - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["slice_id"] == "slice-1" - - -# --------------------------------------------------------------------------- -# Issue #2439: spawner ``base_branch`` matches the slice forest's parent edge -# --------------------------------------------------------------------------- - - -@pytest.mark.skipif(not _HAS_FLASK, reason="Flask not available") -class TestRestartAgentEndpointBaseBranch: - """Restart route forwards the right ``base_branch`` to the spawner (#2439). - - ``gateway.create_worktrees`` is idempotent and ``restart_agent_job`` - passes ``preserve_worktree_on_failure=True``, so most restarts hit a - pre-existing worktree and the wrong ``base_branch`` is never - consulted — but a worktree-absent restart (cleanup race, manual - gateway prune, container hostpath wiped) reaches the worktree - creation path. Forking from ``pipeline.branch`` (the integration - tip) at that point pulls sibling slices' commits into the rebuilt - worktree. - """ - - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_pipeline_level_restart_uses_pipeline_base_branch( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client - ): - """Pipeline-level restart forwards ``pipeline.base_branch`` (not ``pipeline.branch``). - - Every other call site in the codebase uses ``pipeline.base_branch`` - for spawner ``base_branch``; the operator-triggered restart route - was the lone outlier (#2439). - """ - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.base_branch = "main" - - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner - - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart", - json={"reason": "Pipeline-level restart"}, - ) - - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["base_branch"] == "main" - # Defensive: the integration tip must NEVER be forwarded as - # ``base_branch``. If the worktree is rebuilt, forking from - # the tip leaks tip commits into the per-agent worktree. - assert restart_call.kwargs["base_branch"] != pipeline.branch - - @patch("egg_contracts.loader.load_contract") - @patch("routes.pipelines.get_pipeline_state_lock") - @patch("routes.pipelines.get_container_spawner") - @patch("routes.pipelines._resolve_pipeline") - @patch("routes.pipelines.get_repo_path") - def test_root_slice_restart_uses_pipeline_base_branch( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_load_contract, - client, - ): - """A root slice (no parent in the slice forest) falls back to ``pipeline.base_branch``.""" - from egg_contracts.models import Contract, Slice - - mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - pipeline.base_branch = "main" - mock_store = MagicMock() - mock_store.load_pipeline.return_value = pipeline - mock_resolve.return_value = (mock_store, pipeline) - mock_load_contract.return_value = Contract( - pipeline_id="issue-100", - # slice-1 has no dependencies → root of the forest. - slices=[Slice(id="slice-1", name="Root slice")], - ) +def _make_pipeline_with_slice_agents(role, slice_records, pipeline_id="issue-100"): + """Pipeline whose current phase carries per-slice ``AgentExecution`` records. - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-1-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 - mock_spawner_fn.return_value = mock_spawner + ``slice_records`` is a list of ``(slice_id, status)`` tuples — one per + slice that has run (or is running) ``role``. Exercises the #2759 slice + auto-derivation in ``restart_agent``. + """ + pipeline = Pipeline( + id=pipeline_id, + issue_number=100, + repo="owner/repo", + branch="egg/issue-100/work", + status=PipelineStatus.RUNNING, + current_phase=PipelinePhase.IMPLEMENT, + ) + pipeline.phases = { + "implement": PhaseExecution( + phase=PipelinePhase.IMPLEMENT, + status=PipelineStatus.RUNNING, + agents=[ + AgentExecution( + role=role, + status=status, + container_id=f"container-{sid}", + slice_id=sid, + ) + for sid, status in slice_records + ], + ), + } + return pipeline - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-1", - json={"reason": "Root slice agent stalled"}, - ) - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["base_branch"] == "main" +@pytest.mark.skipif(not _HAS_FLASK, reason="Flask not available") +class TestRestartAgentEndpointSliceDerivation: + """Omitted ``slice_id`` is derived from phase agent records (#2759). + + A slice-mode restart that omits ``slice_id`` would otherwise spawn the + agent pipeline-level — ``EGG_SLICE_ID`` unset — so its BRC signals + route to the bare pipeline tracker instead of the slice's tracker and + the slice's consensus wedges with no message-bus recovery path. + """ @patch("egg_contracts.loader.load_contract") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_child_slice_restart_uses_parent_slice_integration_branch( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_load_contract, - client, + def test_omitted_slice_id_derived_from_single_non_complete_record( + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_load_contract, client ): - """A child slice's restart forks from its parent slice's integration branch. - - Mirrors :func:`_run_one_slice_inner`'s ``parent_branch`` - derivation: when ``slice-2`` lists ``slice-1`` as its - dependency, the parent integration branch is - ``/slice-1``. Without this, a worktree-absent - restart of ``slice-2`` would re-fork from the pipeline tip - and pull sibling-slice commits into ``slice-2``'s rebuilt - worktree (#2439). - """ + """One non-complete slice record → its slice_id is derived and forwarded.""" from egg_contracts.models import Contract, Slice mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - pipeline.base_branch = "main" + # slice-1 finished cleanly; slice-2's reviewer crashed. + pipeline = _make_pipeline_with_slice_agents( + AgentRole.REVIEWER_CODE_HOLISTIC, + [ + ("slice-1", AgentExecutionStatus.COMPLETE), + ("slice-2", AgentExecutionStatus.FAILED), + ], + ) mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline mock_resolve.return_value = (mock_store, pipeline) mock_load_contract.return_value = Contract( pipeline_id="issue-100", - slices=[ - Slice(id="slice-1", name="Parent slice"), - Slice( - id="slice-2", - name="Child slice", - dependencies=["slice-1"], - ), - ], + slices=[Slice(id="slice-1", name="First"), Slice(id="slice-2", name="Second")], ) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-2", - json={"reason": "Child slice agent stalled"}, + "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", + json={"reason": "container crashed"}, ) assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - # The agent's assigned branch is still its OWN integration - # branch (slice-2). It's the spawner's ``base_branch`` — the - # ref the per-agent worktree is forked off of when rebuilt — - # that should reference the parent slice's integration branch. - assert restart_call.kwargs["branch"] == "egg/issue-100/slice-2" - assert restart_call.kwargs["base_branch"] == "egg/issue-100/slice-1" + mock_spawner.restart_agent_container.assert_not_called() + # The crashed slice is derived: consensus reset + the live-Job + # label filter + the restart-count bucket are all scoped to + # slice-2, so the event loop respawns the slice-2-scoped agent + # and it rejoins slice-2's BRC tracker rather than the bare one. + from kubernetes_client import LABEL_SLICE_ID + + assert ( + mock_spawner.k8s.list_containers.call_args.kwargs["labels"][LABEL_SLICE_ID] == "slice-2" + ) + assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-2" + assert response.get_json()["data"]["slice_id"] == "slice-2" - @patch("egg_contracts.loader.load_contract") - @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_slice_restart_with_unloadable_contract_falls_back_to_pipeline_base( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_load_contract, - client, + def test_omitted_slice_id_ambiguous_rejected( + self, mock_repo, mock_resolve, mock_spawner_fn, client ): - """When the contract can't be loaded, slice restart falls back to ``pipeline.base_branch``. - - The slice-id existence check (#2421) intentionally falls - through silently when the contract can't be loaded so legitimate - restarts on a pruned worktree aren't gated. The base-branch - fix (#2439) inherits that policy: with no contract to consult - for the parent edge, the spawner gets ``pipeline.base_branch`` - — the same fallback the other call sites use. - """ - from egg_contracts.loader import ContractNotFoundError - + """Multiple non-complete slice records → 400 with the candidate list.""" mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - pipeline.base_branch = "main" + pipeline = _make_pipeline_with_slice_agents( + AgentRole.REVIEWER_CODE_HOLISTIC, + [ + ("slice-1", AgentExecutionStatus.COMPLETE), + ("slice-2", AgentExecutionStatus.FAILED), + ("slice-3", AgentExecutionStatus.RUNNING), + ], + ) mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline mock_resolve.return_value = (mock_store, pipeline) - - mock_load_contract.side_effect = ContractNotFoundError( - "issue-100", Path("/repo/.egg-state/contracts/issue-100.json") - ) - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-2", - json={"reason": "Slice restart with no contract"}, + "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", + json={}, ) - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - assert restart_call.kwargs["base_branch"] == "main" + assert response.status_code == 400 + body = response.get_json() + assert body["success"] is False + assert body["reason"] == "slice_id_required" + assert body["details"]["restart_candidates"] == ["slice-2", "slice-3"] + assert body["details"]["known_slices"] == ["slice-1", "slice-2", "slice-3"] + # Never silently spawn an unscoped agent on an ambiguous restart. + mock_spawner.restart_agent_container.assert_not_called() - @patch("egg_contracts.loader.load_contract") - @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_child_slice_restart_prefers_parent_branch_at_creation( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_load_contract, - client, + def test_omitted_slice_id_all_complete_rejected( + self, mock_repo, mock_resolve, mock_spawner_fn, client ): - """Restart prefers the recorded ``parent_branch_at_creation`` over reconstruction. - - Set by ``_run_one_slice_inner`` when the slice was provisioned - (#2137 TASK-4-2); per the docstring on - ``Slice.parent_branch_at_creation`` it's *the* recorded fact - about how the slice was forked. Reconstructing - ``f"{_issue_branch}/{parent_slice_id}"`` would silently drift - if a future qualifier-suffix or namespacing change lands in - ``_run_one_slice_inner`` but not in the restart route — the - recorded value would not (#2460 review observation 2). - """ - from egg_contracts.models import Contract, Slice - + """All slice records complete → 400 (no unambiguous restart target).""" mock_repo.return_value = "/repo" - mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - pipeline.base_branch = "main" + pipeline = _make_pipeline_with_slice_agents( + AgentRole.REVIEWER_CODE_HOLISTIC, + [ + ("slice-1", AgentExecutionStatus.COMPLETE), + ("slice-2", AgentExecutionStatus.COMPLETE), + ], + ) mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline mock_resolve.return_value = (mock_store, pipeline) - - # The recorded value differs from what reconstruction would - # produce (a hypothetical future qualifier suffix). The route - # must prefer the recorded value over reconstruction. - recorded_parent_branch = "egg/issue-100/slice-1-future-qualifier" - mock_load_contract.return_value = Contract( - pipeline_id="issue-100", - slices=[ - Slice(id="slice-1", name="Parent slice"), - Slice( - id="slice-2", - name="Child slice", - dependencies=["slice-1"], - parent_branch_at_creation=recorded_parent_branch, - ), - ], - ) - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container - mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-2", - json={"reason": "Child slice agent stalled"}, + "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart", + json={}, ) - assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - # The recorded parent branch wins over reconstructed - # ``egg/issue-100/slice-1``. - assert restart_call.kwargs["base_branch"] == recorded_parent_branch + assert response.status_code == 400 + body = response.get_json() + assert body["reason"] == "slice_id_required" + assert body["details"]["restart_candidates"] == [] + mock_spawner.restart_agent_container.assert_not_called() - @patch("egg_contracts.loader.load_contract") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_child_slice_restart_falls_back_when_parent_complete( - self, - mock_repo, - mock_resolve, - mock_spawner_fn, - mock_lock_fn, - mock_load_contract, - client, + def test_explicit_slice_id_bypasses_derivation( + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """Restart falls back to ``pipeline.base_branch`` when the parent slice is complete (#2470). - - When the parent slice has reached ``SliceStatus.COMPLETE``, - its PR has plausibly been merged and the head branch deleted - by GitHub's standard auto-cleanup. The gateway's per-repo - ``git fetch origin `` would then wedge the - restart on a missing-branch fetch error. The route detects - this via the contract's slice status and falls back to - ``pipeline.base_branch`` — equivalent to the - contract-unloadable fall-through (#2421/#2439): prefer - letting the restart proceed over over-strict gating. - - The recorded ``parent_branch_at_creation`` is set on the - child slice here to confirm the fallback wins even when the - recorded value is available (the issue here is the *branch's* - existence, not the route's knowledge of its name). + """An explicit slice_id wins even when records would derive otherwise. + + Derivation runs only when ``slice_id`` is absent; an explicit + value (even one with no agent record) is forwarded as-is, so this + path does not regress the #2410 explicit-scope contract. """ - from egg_contracts.models import Contract, Slice, SliceStatus + from egg_contracts.models import Contract, Slice mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() - pipeline = _make_pipeline_with_running_agent() - pipeline.branch = "egg/issue-100/work" - pipeline.base_branch = "main" + pipeline = _make_pipeline_with_slice_agents( + AgentRole.REVIEWER_CODE_HOLISTIC, + [("slice-2", AgentExecutionStatus.FAILED)], + ) mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline mock_resolve.return_value = (mock_store, pipeline) - mock_load_contract.return_value = Contract( - pipeline_id="issue-100", - slices=[ - Slice( - id="slice-1", - name="Parent slice (complete, branch likely deleted on origin)", - status=SliceStatus.COMPLETE, - ), - Slice( - id="slice-2", - name="Child slice", - dependencies=["slice-1"], - parent_branch_at_creation="egg/issue-100/slice-1", - ), - ], - ) - mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-slice-2-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner - response = client.post( - "/api/v1/pipelines/issue-100/agents/coder/restart?slice_id=slice-2", - json={"reason": "Child slice restart after parent merged"}, - ) + with patch("egg_contracts.loader.load_contract") as mock_load_contract: + mock_load_contract.return_value = Contract( + pipeline_id="issue-100", + slices=[Slice(id="slice-1", name="First"), Slice(id="slice-2", name="Second")], + ) + response = client.post( + "/api/v1/pipelines/issue-100/agents/reviewer_code_holistic/restart?slice_id=slice-1", + json={}, + ) assert response.status_code == 200 - restart_call = mock_spawner.restart_agent_container.call_args - # Fallback wins over both the recorded parent branch and - # reconstructed ``egg/issue-100/slice-1``. - assert restart_call.kwargs["base_branch"] == "main" - # Agent's assigned branch is unaffected — still its own - # integration branch. - assert restart_call.kwargs["branch"] == "egg/issue-100/slice-2" + mock_spawner.restart_agent_container.assert_not_called() + # The explicit slice_id wins over derivation: it scopes the count + # bucket and the response. + assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-1" + assert response.get_json()["data"]["slice_id"] == "slice-1" # --------------------------------------------------------------------------- @@ -2118,18 +1291,7 @@ def test_slice_3_restart_does_not_mutate_slice_2_record( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="container-slice-3-RESTARTED", - container_name="egg-issue-100-slice-3-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -2160,7 +1322,9 @@ def test_slice_3_restart_does_not_mutate_slice_2_record( assert slice2_persisted["container_id"] == slice2_container_before, ( "slice-2 container_id must not change when slice-3 is restarted" ) - assert slice3_persisted["container_id"] == "container-slice-3-RESTARTED" + # #3164: the restarted record's container_id is cleared to None — + # the event loop owns the respawn and sets the live container_id. + assert slice3_persisted["container_id"] is None assert slice3_persisted["status"] == AgentExecutionStatus.RUNNING.value @patch("routes.pipelines.get_pipeline_state_lock") @@ -2212,18 +1376,7 @@ def test_slice_3_restart_with_no_existing_record_appends_with_slice_id( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="container-slice-3-NEW", - container_name="egg-issue-100-slice-3-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -2245,7 +1398,9 @@ def test_slice_3_restart_with_no_existing_record_appends_with_slice_id( ) slice3_rows = [a for a in persisted_agents if a["slice_id"] == "slice-3"] assert len(slice3_rows) == 1 - assert slice3_rows[0]["container_id"] == "container-slice-3-NEW" + # #3164: appended record has no resident container — event loop respawns. + assert slice3_rows[0]["container_id"] is None + assert slice3_rows[0]["status"] == AgentExecutionStatus.RUNNING.value # --------------------------------------------------------------------------- @@ -2563,20 +1718,26 @@ def test_restart_lock_initialization(self): @pytest.mark.skipif(not _HAS_FLASK, reason="Flask not available") class TestConsensusResetOrdering: - """Tests that consensus is reset AFTER successful spawn, not before (issue #1695, item 5).""" + """Consensus reset is orchestrator-native and unconditional (#3164). + + The old spawn-then-reset ordering (issue #1695 item 5) is moot: there + is no resident spawn that can fail, so consensus is always reset and + the event loop respawns from the cleared state. + """ @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_spawn_failure_preserves_consensus( + def test_live_job_deletion_failure_still_resets_consensus( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """If spawner raises ContainerSpawnError, consensus state must NOT be reset. + """A best-effort live-Job teardown failure does not block consensus reset. - Regression test for issue #1695 item 5: consensus reset was previously - done before the spawn attempt, leaving orphaned consensus deletion on - spawn failure. + After #3164 the only "failure" before the consensus reset is the + best-effort Job teardown. It must be swallowed so the consensus + reset (and thus the event-loop respawn) still happens — the + opposite of the old "preserve consensus on spawn failure" rule. """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -2587,13 +1748,10 @@ def test_spawn_failure_preserves_consensus( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - mock_spawner.restart_agent_container.side_effect = ContainerSpawnError("Docker error") + mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") + mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner - # Mock the peer-consensus tracker via sys.modules so the inline - # import inside the route handler resolves correctly (no create=True - # needed). The legacy ``consensus`` evaluator reset was removed in - # #2777 (slice-2). mock_tracker = MagicMock() with patch.dict( @@ -2609,19 +1767,22 @@ def test_spawn_failure_preserves_consensus( json={"reason": "Agent stalled"}, ) - assert response.status_code == 500 - - # Consensus should NOT have been reset since spawn failed - mock_tracker.remove_agent.assert_not_called() + assert response.status_code == 200 + mock_spawner.restart_agent_container.assert_not_called() + mock_tracker.remove_agent.assert_called_once_with("coder") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_successful_spawn_resets_consensus( + def test_restart_resets_consensus( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """On successful spawn, consensus state should be reset for the agent.""" + """A restart resets the agent's BRC consensus state (#3164). + + Only the peer-consensus (BRC) tracker is reset; the legacy + ``consensus`` evaluator reset was removed in #2777 (slice-2). + """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -2631,25 +1792,10 @@ def test_successful_spawn_resets_consensus( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner - # Patch the peer-consensus tracker import inside the route handler. - # The legacy ``consensus`` evaluator reset was removed in #2777 - # (slice-2 deleted ``orchestrator/consensus.py``); only the - # peer-consensus (BRC) tracker is reset on restart now. mock_tracker = MagicMock() with patch.dict( @@ -2666,8 +1812,7 @@ def test_successful_spawn_resets_consensus( ) assert response.status_code == 200 - - # Consensus should have been reset after successful spawn + mock_spawner.restart_agent_container.assert_not_called() mock_tracker.remove_agent.assert_called_once_with("coder") @@ -2693,18 +1838,7 @@ def test_successful_restart_calls_reset_agent( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -2734,9 +1868,9 @@ def test_successful_restart_calls_reset_agent( def test_successful_restart_refreshes_started_at( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """``restart_agent`` must overwrite ``agent.started_at`` with the new - spawn timestamp so ``_get_concurrent_status`` reports an - ``elapsed_seconds`` anchored on the live container, not the dead one + """``restart_agent`` must overwrite ``agent.started_at`` with the + restart timestamp so ``_get_concurrent_status`` reports an + ``elapsed_seconds`` anchored on the respawn, not the dead container (issue #2084).""" mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -2751,18 +1885,7 @@ def test_successful_restart_refreshes_started_at( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - new_container = SpawnedContainer( - container_info=ContainerInfo( - container_id="new-container-xyz", - container_name="egg-issue-100-coder", - status=ContainerStatus.RUNNING, - ), - session_info=None, - agent_role=AgentRole.CODER, - pipeline_id="issue-100", - environment={}, - ) - mock_spawner.restart_agent_container.return_value = new_container + mock_spawner.k8s.list_containers.return_value = [] mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner @@ -2786,9 +1909,9 @@ def test_successful_restart_refreshes_started_at( agent = pipeline.phases["implement"].agents[0] assert agent.started_at is not None assert agent.started_at > original_started_at, ( - "restart_agent must refresh started_at to the new container's " - "spawn time so _get_concurrent_status reports an elapsed_seconds " - "anchored on the live container instead of the dead one." + "restart_agent must refresh started_at to the restart time so " + "_get_concurrent_status reports an elapsed_seconds anchored on " + "the respawn instead of the dead container." ) # The refreshed timestamp should be very recent (the test just ran). assert (datetime.now(UTC) - agent.started_at) < timedelta(seconds=30) @@ -2797,11 +1920,18 @@ def test_successful_restart_refreshes_started_at( @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") - def test_failed_spawn_skips_reset_agent( + def test_live_job_deletion_failure_still_resets_health( self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client ): - """If the spawn fails, the heartbeat anchor must be preserved so the - old container's stall signal isn't silently swallowed.""" + """A best-effort live-Job teardown failure does not block the health reset (#3164). + + The respawn is owned by the event loop, so the health-monitor + anchor must be cleared unconditionally — a stuck-pod teardown + error must not leave the dead container's clock anchored against + the respawn (issue #2084). Replaces the old + ``test_failed_spawn_skips_reset_agent`` whose spawn-failure path + no longer exists. + """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() pipeline = _make_pipeline_with_running_agent() @@ -2811,7 +1941,8 @@ def test_failed_spawn_skips_reset_agent( mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() - mock_spawner.restart_agent_container.side_effect = ContainerSpawnError("boom") + mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") + mock_spawner.get_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner mock_hm = MagicMock() @@ -2830,5 +1961,6 @@ def test_failed_spawn_skips_reset_agent( json={"reason": "stalled"}, ) - assert response.status_code == 500 - mock_hm.reset_agent.assert_not_called() + assert response.status_code == 200 + mock_spawner.restart_agent_container.assert_not_called() + mock_hm.reset_agent.assert_called_once_with("coder") diff --git a/orchestrator/tests/test_two_tier_integration.py b/orchestrator/tests/test_two_tier_integration.py index c960769cd5..5a209fc8f6 100644 --- a/orchestrator/tests/test_two_tier_integration.py +++ b/orchestrator/tests/test_two_tier_integration.py @@ -164,6 +164,7 @@ def test_full_flow_stall_to_nudge_to_escalation(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) store = ProgressStore() # Track escalations @@ -334,6 +335,8 @@ def test_escalation_resolves_no_re_escalation(self): config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -368,6 +371,8 @@ def test_escalation_resolves_heartbeat_variant(self): config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -402,6 +407,8 @@ def test_recovery_then_second_stall_re_escalates(self): config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) + escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -452,6 +459,7 @@ def test_two_agents_stall_at_different_times(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER, AGENT_ID_TESTER}) # Register both agents _emit_progress(bus, agent_id=AGENT_ID_CODER) @@ -487,6 +495,7 @@ def test_both_agents_stall_get_independent_escalations(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER, AGENT_ID_TESTER}) # Register both agents _emit_progress(bus, agent_id=AGENT_ID_CODER) @@ -516,6 +525,7 @@ def test_one_agent_resolves_other_stays_escalated(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER, AGENT_ID_TESTER}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -560,6 +570,7 @@ def test_independent_escalation_types(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER, AGENT_ID_TESTER}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -611,6 +622,7 @@ def test_escalation_goes_to_hitl_when_overseer_disabled(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) escalations = [] monitor.on_escalation(lambda e: escalations.append(e)) @@ -702,6 +714,7 @@ def test_container_exit_always_hitl_regardless_of_overseer_setting(self): pipeline_id=PIPELINE_ID, config=config, ) + monitor.set_active_roles({AGENT_ID_CODER}) escalations: list = [] monitor.on_escalation(lambda e, _esc=escalations: _esc.append(e)) diff --git a/orchestrator/tests/test_wait_instruction_ratchet.py b/orchestrator/tests/test_wait_instruction_ratchet.py index 752060c312..45ea7c3f68 100644 --- a/orchestrator/tests/test_wait_instruction_ratchet.py +++ b/orchestrator/tests/test_wait_instruction_ratchet.py @@ -1,17 +1,21 @@ """Blocking-wait instruction ratchet for agent-facing prompt sources (#3157). The event-pump contract (#2908) is: **agents never wait on the message -bus** — the consensus wrapper (`orchestrator/consensus_wrapper.py`) owns -every blocking read (`egg-orch message wait` / `wait-loop`) and invokes -the agent one-shot per actionable BRC event. The per-event prompt -(`orchestrator/routes/event_prompt.py`) states the contract to the agent -directly; `docs/reference/agent-wait-patterns.md` §0 is the reference. +bus**. Since [#3164](https://github.com/jwbron/egg/issues/3164) retired +the in-pod wait arm, the **orchestrator owns all waiting** +(`orchestrator/event_loop.py`): it derives the next BRC event in-process +and spawns the agent one-shot per actionable event — agents never issue +a blocking read (`egg-orch message wait` / `wait-loop`). The per-event +prompt (`orchestrator/routes/event_prompt.py`) states the contract to the +agent directly; `docs/reference/agent-wait-patterns.md` §0 is the +reference. #3157 retired the pre-pump agent-facing canon that still taught the "STAY ALIVE" blocking-wait idiom. This module is the ratchet: any future reintroduction of a blocking-wait instruction into the scanned agent-facing sources fails CI before the contract can drift back — -recreating the #1897 pathologies and double-waiting against the wrapper. +recreating the #1897 pathologies and double-waiting against the +orchestrator-owned loop. Scope (the sources that reach agents): @@ -24,8 +28,9 @@ Patterns matched: * `egg-orch message wait` / `egg-orch message wait-loop` — the blocking - CLI surface is wrapper-internal; agent-facing text may name it only to - forbid it (those occurrences are allowlisted with justification). + CLI surface is not for agents (the orchestrator owns all waiting); + agent-facing text may name it only to forbid it (those occurrences are + allowlisted with justification). * `STAY ALIVE` / `stay-alive` / `stay_alive` — the retired idiom name. Agent-facing text must not present it as a live step. @@ -90,23 +95,23 @@ # names the forbidden command in order to forbid it. _ALLOWLIST: tuple[tuple[str, str, int], ...] = ( # JUSTIFICATION: mission.md tells the agent it does NOT need to call - # `egg-orch message wait` because the wrapper owns sequencing — the - # token appears only inside the negative instruction that codifies - # the event-pump contract (#2908 slice-3 mission rewrite). + # `egg-orch message wait` because the orchestrator owns all waiting — + # the token appears only inside the negative instruction that codifies + # the event-pump contract (#2908 mission rewrite; #3164 ownership flip). ("sandbox/agent-config/rules/mission.md", "egg-orch message wait", 155), # JUSTIFICATION: the rules/orchestrator.md CLI-reference callout # forbids agent-tier blocking waits (#3157); it names - # `egg-orch message wait` / `wait-loop` only to mark them - # wrapper-internal and direct the agent to exit instead. Two - # matches on the same line (the wrapper attribution and the - # "do NOT run" clause) — both covered by this entry. + # `egg-orch message wait` / `wait-loop` only to mark them not for + # agents (the orchestrator owns the wait, #3164) and direct the agent + # to exit instead. Two matches on the same line (the orchestrator + # attribution and the "do NOT run" clause) — both covered by this entry. ("sandbox/agent-config/rules/orchestrator.md", "egg-orch message wait", 64), # JUSTIFICATION: the per-event prompt contract string is the # authoritative agent-facing wording of the invariant — "Do NOT - # block on ``egg-orch message wait-loop`` yourself: the wrapper - # owns the wait and the heartbeat (#2908 slice-2)". The token is - # present precisely to forbid the call. - ("orchestrator/routes/event_prompt.py", "egg-orch message wait-loop", 829), + # block on ``egg-orch message wait-loop`` yourself: the orchestrator + # owns the wait and spawns you one-shot per event (#3164)". The token + # is present precisely to forbid the call. + ("orchestrator/routes/event_prompt.py", "egg-orch message wait-loop", 827), ) @@ -161,15 +166,16 @@ def _filter_allowlist( ) def test_agent_facing_markdown_has_no_wait_instructions(prompt_path: Path) -> None: """No agent-facing markdown source may instruct a blocking bus wait - or present the STAY ALIVE idiom as live. The wrapper owns all - waiting (#2908); agents handle one event and exit (#3157).""" + or present the STAY ALIVE idiom as live. The orchestrator owns all + waiting (#2908/#3164); agents handle one event and exit (#3157).""" text = prompt_path.read_text(encoding="utf-8") findings = _filter_allowlist(prompt_path.relative_to(_REPO_ROOT).as_posix(), _scan_text(text)) assert not findings, ( f"blocking-wait instruction in {prompt_path.name}: {findings}; " - f"agents never wait on the bus — the consensus wrapper owns " - f"`egg-orch message wait[-loop]` and invokes agents one-shot " - f"per event (docs/reference/agent-wait-patterns.md §0). If the " + f"agents never wait on the bus — the orchestrator owns all " + f"waiting (it derives `egg-orch message wait[-loop]` in-process " + f"and spawns agents one-shot per event; " + f"docs/reference/agent-wait-patterns.md §0). If the " f"occurrence is a negative instruction (naming the command to " f"forbid it), add a justified _ALLOWLIST entry." ) diff --git a/sandbox/agent-config/rules/mission.md b/sandbox/agent-config/rules/mission.md index 2fd8a78f7b..ffa5df8419 100644 --- a/sandbox/agent-config/rules/mission.md +++ b/sandbox/agent-config/rules/mission.md @@ -144,24 +144,24 @@ including your role type (producer/reviewer), active agent roster, assigned reviewers or producers, preparation steps, and the exact consensus commands to run. Follow those instructions exactly. -### You are an event handler — the wrapper owns the wait +### You are an event handler — the orchestrator owns the wait -Under the BRC event-pump wrapper ([#2908](https://github.com/jwbron/egg/issues/2908), the sole production path since slice-4 deleted the legacy capped-restart wrapper), the **bash wrapper around your invocation owns the BRC wait, not you**. You are invoked one-shot per actionable event: +Under the BRC event loop ([#2908](https://github.com/jwbron/egg/issues/2908), with the in-pod wait arm retired by [#3164](https://github.com/jwbron/egg/issues/3164)), the **orchestrator owns the BRC wait, not you**. The orchestrator derives the next actionable BRC event in-process and spawns you one-shot per event: 1. **Act on the single event** named in your prompt — review the proposal, fix the NACKs, confirm consensus, whatever the event called for. 2. **Update your BRC memory file** at `.egg-state/agent-outputs//brc-memory-.md` so the next invocation can re-enter with continuity (writes happen automatically inside `brc_ack` / `brc_nack`; see `$EGG_REPO_PATH/docs/architecture/brc-memory.md`). A `brc-memory.md` without the pipeline-id suffix is a previous pipeline's leftover — ignore it. -3. **Exit naturally** when your handling of the event is complete. The wrapper loops back to `egg-orch brc next-action` and invokes you again when the next actionable event arrives, or calls `egg-orch consensus confirmed` and exits 0 once `brc next-action` returns the `complete` action (i.e. the role is marked complete in `consensus status`). +3. **Exit naturally** when your handling of the event is complete. The orchestrator derives the next action via `egg-orch brc next-action` and spawns you again when the next actionable event arrives, or handles confirm/complete orchestrator-side once the role is marked complete in `consensus status`. -You do **not** need to call `egg-orch message wait` yourself, hold a polling loop between events, thread cursors across re-entries, or guard against early exit — the wrapper-side deterministic bash loop is the single source of truth for sequencing. Clean exit after handling your event is the expected lifecycle, not a failure mode. +You do **not** need to call `egg-orch message wait` yourself, hold a polling loop between events, thread cursors across re-entries, or guard against early exit — the orchestrator's in-process event loop is the single source of truth for sequencing. Clean exit after handling your event is the expected lifecycle, not a failure mode. ### Key Principles - The orchestrator *observes* consensus, it doesn't *decide* it - **Producers**: orient → work → propose → on each invocation, respond to reviews or confirm consensus - **Reviewers**: on each invocation, prepare → review the named proposal → ACK/NACK (or re-confirm on `CONSENSUS_RE_REVIEW`) -- **Dual-role agents** (e.g. `tester`): handle both the producer-side and reviewer-side events for the invocation, in the order the wrapper dispatches them +- **Dual-role agents** (e.g. `tester`): handle both the producer-side and reviewer-side events for the invocation, in the order the orchestrator dispatches them - **Adversarial re-review** of a producer's v2+ delta is a fresh review — read the per-producer `git log {last_reviewed_commit_sha}..HEAD --not origin/{base_branch} -p` delivered in your prompt; the durable BRC memory file under `.egg-state/agent-outputs//` carries the prior verdict so you can compare without re-reading the codebase end-to-end -- See `$EGG_REPO_PATH/docs/architecture/orchestrator.md` (BRC Event-Pump Wrapper section) for the wrapper-side lifecycle and `$EGG_REPO_PATH/docs/reference/agent-wait-patterns.md` §10 for the wait surface +- See `$EGG_REPO_PATH/docs/architecture/orchestrator.md` (BRC Consensus Wrapper section) for the orchestrator-owned event-loop lifecycle and `$EGG_REPO_PATH/docs/reference/agent-wait-patterns.md` §10 for the wait surface > **Note.** The collapsed preamble above is shared across BRC roles — `_build_brc_preamble` renders it unconditionally. The event-handler contract above is the **sole** production behaviour: slice-4 of [#2908](https://github.com/jwbron/egg/issues/2908) deleted the legacy capped-restart template and the `EGG_BRC_EVENT_PUMP` selector from `orchestrator/consensus_wrapper.py` (which now holds only the event-pump wrapper), so there is no alternate path. Your live event prompt is always authoritative — follow it over this meta-reference if the two ever diverge. diff --git a/sandbox/agent-config/rules/orchestrator.md b/sandbox/agent-config/rules/orchestrator.md index 91b4a3198f..a5f5baa4c1 100644 --- a/sandbox/agent-config/rules/orchestrator.md +++ b/sandbox/agent-config/rules/orchestrator.md @@ -61,7 +61,7 @@ BRC consensus + heartbeats: - `mcp__brc__confirm` — Prefer this over `egg-orch consensus confirmed`. Producer confirms after all reviewers ACK. Returns `ok: True` only when the producer transitioned to CONFIRMED; on `ok: False` (status `pending_acks`) the transition was rejected — read `message` for the reason (e.g. `producer_not_fully_acked`, `global_zero_proposal`, `stale_acks`) and take corrective action before retrying. - `mcp__brc__send_heartbeat` — Prefer this over `egg-orch message heartbeat`. Emit a structured HEARTBEAT to the dedicated `/heartbeat` endpoint. -> **Never issue blocking waits on the message bus** (#2908, #3157). The consensus wrapper owns all waiting: it blocks in `egg-orch message wait-loop` between BRC events, emits the heartbeats while blocked, and invokes you one-shot per actionable event. When you have acted (proposed, ACKed, NACKed, or confirmed), exit naturally — do NOT run `egg-orch message wait` / `wait-loop`, an outer poll loop, or a `sleep` yourself. The contract is in `docs/reference/agent-wait-patterns.md` §0. (The pre-#2908 MCP blocking-wait tools were removed in #2211 and are not coming back — long polls don't fit the MCP transport caps.) +> **Never issue blocking waits on the message bus** (#2908, #3157, #3164). The orchestrator owns all waiting: it derives the next BRC event in-process and spawns you one-shot per actionable event (the in-pod wait arm was retired by #3164). When you have acted (proposed, ACKed, NACKed, or confirmed), exit naturally — do NOT run `egg-orch message wait` / `wait-loop`, an outer poll loop, or a `sleep` yourself. The contract is in `docs/reference/agent-wait-patterns.md` §0. (The pre-#2908 MCP blocking-wait tools were removed in #2211 and are not coming back — long polls don't fit the MCP transport caps.) Progress + overseer (iter-2 added the overseer surface): diff --git a/sandbox/egg_agent_tools/tools/message.py b/sandbox/egg_agent_tools/tools/message.py index 98fca8a5f3..d08744c849 100644 --- a/sandbox/egg_agent_tools/tools/message.py +++ b/sandbox/egg_agent_tools/tools/message.py @@ -8,12 +8,13 @@ removed in #2211 — long-poll waits don't fit the MCP transport (the in-process SDK caps tool calls at ~60 s and the streamable-HTTP MCP caps at ~30 s), and the cap-elapsed return is a full LLM turn. No -replacement exists at the agent tier by design (#2908, #3157): the -consensus wrapper owns every blocking wait on the bus — it blocks in -``egg-orch message wait-loop`` between BRC events and invokes the -agent one-shot per actionable event. Agents act on the single event -in their prompt and exit; they never wait. See §0 of -``docs/reference/agent-wait-patterns.md`` for the contract. +replacement exists at the agent tier by design (#2908, #3157, #3164): +the orchestrator owns every blocking wait on the bus — it derives the +next BRC event in-process and spawns the agent one-shot per actionable +event (the in-pod ``egg-orch message wait-loop`` arm was retired by +#3164). Agents act on the single event in their prompt and exit; they +never wait. See §0 of ``docs/reference/agent-wait-patterns.md`` for +the contract. The remaining tool lives under the ``brc`` namespace; it renders as ``mcp__brc__send_heartbeat``. @@ -67,8 +68,8 @@ "rate-limited) to the dedicated /heartbeat endpoint on a state " "transition (WORKING, WAITING_ON_ROLE + waiting_on=, PROPOSED, " "IDLE). Prefer this over 'egg-orch message heartbeat'. Do not emit " - "WAITING_FOR_EVENT — the consensus wrapper owns bus waits and their " - "heartbeats; never block waiting on BRC yourself.", + "WAITING_FOR_EVENT — the orchestrator owns all waiting and spawns you " + "one-shot per event; never block waiting on BRC yourself.", _HEARTBEAT_SCHEMA, ) async def brc_send_heartbeat(args: dict[str, Any]) -> dict[str, Any]: From 39e968cd379eae2397b1117da923c917231a2797 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:33:08 +0000 Subject: [PATCH 2/3] fix(orchestrator): restore restart budget + respawn on restart_agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review on #3244 (retire in-pod BRC wait arm). Blocking 1 — restart_agent on a FAILED/CANCELLED pipeline was a false- success no-op: the event loop and _run_pipeline driver thread are already dead on those paths, so resetting consensus left the pipeline RUNNING-but- idle with nothing to respawn it. The route now relaunches a fresh _run_pipeline driver thread (mirroring restart_phase step 7) when it makes the FAILED/CANCELLED -> RUNNING transition, restarting the event loop which then performs the respawn. A RUNNING pipeline (live loop) is untouched. Blocking 2 — the per-(pipeline, role, slice) restart cap and restart_count telemetry were both lost when the route stopped calling restart_agent_job. Extracted the check+increment into KubernetesSpawner. check_and_increment_restart_count (shared with restart_agent_job via _apply_restart_budget); the route now enforces it before any destructive action, rejecting an over-budget restart with HTTP 429 instead of a misleading success, and reports the real incremented count. Tests: assert the driver-thread relaunch actually happens on FAILED/ CANCELLED, that a RUNNING restart does not relaunch, that an over-budget restart is rejected 429 without mutating state, and cover the new spawner helper (increment, cap, slice scoping). --- orchestrator/kubernetes_spawner.py | 87 +++++++++-- orchestrator/routes/pipelines.py | 107 +++++++++---- orchestrator/tests/test_restart_agent.py | 183 ++++++++++++++++++----- 3 files changed, 300 insertions(+), 77 deletions(-) diff --git a/orchestrator/kubernetes_spawner.py b/orchestrator/kubernetes_spawner.py index 005680d431..0da56480c5 100644 --- a/orchestrator/kubernetes_spawner.py +++ b/orchestrator/kubernetes_spawner.py @@ -2384,6 +2384,74 @@ def cleanup_pipeline( return removed + def _apply_restart_budget( + self, + restart_key: tuple[str, str, str | None], + max_restarts: int, + ) -> int: + """Check + increment the restart budget for ``restart_key``. + + The caller MUST already hold ``_get_restart_lock(restart_key)`` so + the read-modify-write of ``_restart_counts`` is atomic. Raises + :class:`KubernetesSpawnError` when the budget is already exhausted; + otherwise increments the count and returns the new value. + """ + pipeline_id, agent_role_value, _slice_id = restart_key + current_count = self._restart_counts.get(restart_key, 0) + if current_count >= max_restarts: + raise KubernetesSpawnError( + f"Restart limit ({max_restarts}) exceeded for {agent_role_value} " + f"in pipeline {pipeline_id} (restarted {current_count} times)" + ) + self._restart_counts[restart_key] = current_count + 1 + return current_count + 1 + + def check_and_increment_restart_count( + self, + pipeline_id: str, + agent_role: AgentRole, + slice_id: str | None = None, + max_restarts: int = 2, + ) -> int: + """Atomically enforce + bump the per-(pipeline, role, slice) budget. + + Extracted from :meth:`restart_agent_job` (#3244). After #3164 moved + respawn ownership to the orchestrator event loop, the + ``restart_agent`` route no longer calls ``restart_agent_job`` and so + must enforce the restart budget itself before delegating the respawn. + Acquires the per-key restart lock, raises + :class:`KubernetesSpawnError` when the budget is exhausted, otherwise + increments and returns the new count. The increment-before-respawn + semantics match the in-band check ``restart_agent_job`` performs, so + an operator/overseer cannot reset consensus an unbounded number of + times on a converging phase. + + Args: + pipeline_id: Pipeline ID. + agent_role: Agent role being restarted. + slice_id: Optional slice scope (#2410). Slice-scoped restarts get + an independent budget bucket keyed on the slice. + max_restarts: Maximum restart attempts per agent per phase. + + Returns: + The new restart count after incrementing. + + Raises: + KubernetesSpawnError: If the budget is exhausted or the per-key + restart lock cannot be acquired. + """ + restart_key = (pipeline_id, agent_role.value, slice_id) + lock = self._get_restart_lock(restart_key) + if not lock.acquire(timeout=120): + raise KubernetesSpawnError( + f"Timed out waiting to acquire restart lock for " + f"{agent_role.value} in pipeline {pipeline_id}" + ) + try: + return self._apply_restart_budget(restart_key, max_restarts) + finally: + lock.release() + def restart_agent_job( self, pipeline_id: str, @@ -2476,16 +2544,11 @@ def restart_agent_job( f"{agent_role.value} in pipeline {pipeline_id}" ) try: - current_count = self._restart_counts.get(restart_key, 0) - - if current_count >= max_restarts: - raise KubernetesSpawnError( - f"Restart limit ({max_restarts}) exceeded for {agent_role.value} " - f"in pipeline {pipeline_id} (restarted {current_count} times)" - ) - - # Increment count before spawn so failed attempts burn a restart budget slot - self._restart_counts[restart_key] = current_count + 1 + # Increment count before spawn so failed attempts burn a restart + # budget slot. Shared with the orchestrator-native ``restart_agent`` + # route via ``check_and_increment_restart_count`` (#3244) so both + # paths enforce the same per-(pipeline, role, slice) cap. + new_count = self._apply_restart_budget(restart_key, max_restarts) # ``job_name`` matches the gateway session container_id used at # spawn time (hyphenated, no JOB_PREFIX); ``actual_k8s_job_name`` @@ -2502,7 +2565,7 @@ def restart_agent_job( "Restarting agent Job", pipeline_id=pipeline_id, role=agent_role.value, - restart_count=current_count + 1, + restart_count=new_count, max_restarts=max_restarts, reason=reason, ) @@ -2623,7 +2686,7 @@ def restart_agent_job( pipeline_id=pipeline_id, role=agent_role.value, new_job_name=spawned.container_info.job_name, - restart_count=current_count + 1, + restart_count=new_count, ) return spawned diff --git a/orchestrator/routes/pipelines.py b/orchestrator/routes/pipelines.py index 0a408b22dd..7b9a9f53ee 100644 --- a/orchestrator/routes/pipelines.py +++ b/orchestrator/routes/pipelines.py @@ -2657,6 +2657,9 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: immediately log FATAL and ``exit 64``, so ``restart_agent`` no longer spawns anything itself. Instead it: + 0. Enforces the per-(pipeline, role, slice) restart budget + (``check_and_increment_restart_count``); a request over budget is + rejected with HTTP 429 before any state is mutated (#3244). 1. Best-effort deletes the role's live one-shot Job(s) (to kill a stuck pod). One-shot Jobs carry an event-discriminator suffix in their name, so they are found by label @@ -2665,10 +2668,15 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: 2. Resets the role's consensus state and health-monitor anchor. 3. Marks the agent record RUNNING with ``container_id = None``. - The event loop (polling ~every 5s during the concurrent phase) then - spawns a fresh one-shot pod once the role's consensus state is - reset — that is the respawn. The agent's per-agent worktree is - preserved so committed work is retained. + For a pipeline that is already RUNNING, the live event loop (polling + ~every 5s during the concurrent phase) spawns a fresh one-shot pod once + the role's consensus state is reset — that is the respawn. For a pipeline + that was FAILED/CANCELLED the event loop and its ``_run_pipeline`` driver + thread are already dead, so the route also relaunches a fresh driver + thread (mirroring ``restart_phase``) to restart the event loop; otherwise + the reset would leave the pipeline RUNNING-but-idle with nothing to + respawn it (#3244). The agent's per-agent worktree is preserved so + committed work is retained. URL params: pipeline_id: Pipeline ID @@ -2900,12 +2908,47 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: current_phase = pipeline.current_phase.value phase_exec = pipeline.phases.get(current_phase) + # Enforce the per-(pipeline, role, slice) restart budget BEFORE any + # destructive action (#3244 review). Pre-#3164 this cap lived inside + # ``restart_agent_job``, which the route no longer calls — without + # re-enforcing it here an operator/overseer could call ``restart_agent`` + # without bound, each call resetting consensus and actively preventing a + # live phase from converging. ``check_and_increment_restart_count`` raises + # when the budget is exhausted; reject loudly (429) instead of flipping + # status / resetting consensus and returning a misleading success. The + # returned count is the source of truth for the ``restart_count`` + # telemetry below (the old read-only ``get_restart_count`` read always + # reported 0 on this path since nothing incremented it). + try: + new_restart_count = spawner.check_and_increment_restart_count( + pipeline_id, role, slice_id=slice_id + ) + except KubernetesSpawnError as budget_err: + logger.warning( + "restart_agent rejected: restart budget exhausted", + pipeline_id=pipeline_id, + agent_role=agent_role, + slice_id=slice_id, + error=str(budget_err), + ) + return make_error_response(str(budget_err), status_code=429) + # Early status update: transition FAILED/CANCELLED -> RUNNING so that - # get_status returns "running" immediately and the event loop (which - # only acts on RUNNING pipelines) is unblocked to respawn (see #1594, - # #1725). With no resident spawn (#3164) there is no failure path that - # would revert this transition. - if pipeline.status in (PipelineStatus.FAILED, PipelineStatus.CANCELLED): + # get_status returns "running" immediately. Unlike a RUNNING pipeline — + # whose live event loop picks up the consensus reset below and respawns + # within one poll — a FAILED/CANCELLED pipeline has NO live event loop: + # ``_run_concurrent_phase`` already returned and ``stop_event_loop()`` + # tore the loop down on its way out, and the ``_run_pipeline`` driver + # thread has exited. Resetting consensus alone would leave the pipeline + # RUNNING-but-idle with nothing to respawn it (#3244 review). So when we + # make this transition we record it and relaunch a fresh ``_run_pipeline`` + # driver thread at the end of the route (mirroring ``restart_phase`` step + # 7) — that restarts the event loop, which then performs the respawn. + pipeline_was_inactive = pipeline.status in ( + PipelineStatus.FAILED, + PipelineStatus.CANCELLED, + ) + if pipeline_was_inactive: early_lock = get_pipeline_state_lock(pipeline_id) with early_lock: pipeline = store.load_pipeline(pipeline_id) @@ -2914,8 +2957,18 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: _phase_exec = pipeline.phases.get(current_phase) if _phase_exec is not None: _phase_exec.status = PipelineStatus.RUNNING + # Bump run_epoch so the relaunched driver thread (below) owns a + # fresh epoch namespace and any stale thread that observes the + # transition detects itself as superseded (mirrors + # ``restart_phase`` / ``advance_phase``). + pipeline.run_epoch = datetime.now(UTC) pipeline.updated_at = datetime.now(UTC) store.update_pipeline(pipeline_id, pipeline.model_dump(mode="json")) + else: + # Lost the race — another writer already moved it off + # FAILED/CANCELLED, so its driver thread / event loop is + # live and will own the respawn. Don't relaunch a duplicate. + pipeline_was_inactive = False # #3164: ``restart_agent`` no longer spawns a resident pod. The # orchestrator event loop owns the BRC respawn — once the role's @@ -3086,30 +3139,34 @@ def restart_agent(pipeline_id: str, agent_role: str) -> tuple[Response, int]: pipeline.updated_at = datetime.now(UTC) store.update_pipeline(pipeline_id, pipeline.model_dump(mode="json")) - # Slice-scoped restarts (#2410) read the per-slice budget bucket - # ``(pipeline_id, agent_role, slice_id)``; the pipeline-level - # ``(pipeline_id, agent_role, None)`` bucket is untouched. Reading - # without ``slice_id`` here would return the pipeline-level count - # (typically zero) and the JSON response below would misreport the - # operator's "you've burned N of M restarts" telemetry. Best-effort: - # the count is informational, so a lookup failure must not fail the - # restart. + # ``restart_count`` is the value just incremented by + # ``check_and_increment_restart_count`` above (#3244). It is scoped to the + # same ``(pipeline_id, agent_role, slice_id)`` bucket the cap is enforced + # on, so it correctly reports the operator's "you've burned N of M + # restarts" telemetry — the pre-fix read-only ``get_restart_count`` read + # always reported 0 here because nothing on this path incremented it. response_data: dict[str, object] = { "agent_role": agent_role, "slice_id": slice_id, "respawn": "delegated to orchestrator event loop", + "restart_count": new_restart_count, } - try: - response_data["restart_count"] = spawner.get_restart_count( - pipeline_id, agent_role, slice_id=slice_id - ) - except Exception as count_err: # noqa: BLE001 - informational only - logger.warning( - "Failed to read restart_count for restart response", + + # When the pipeline was FAILED/CANCELLED its event loop and driver thread + # are dead (see the early-status comment above), so the consensus reset + # alone has nothing to act on it. Relaunch a fresh ``_run_pipeline`` driver + # thread — exactly as ``restart_phase`` step 7 does — to restart the event + # loop, which then respawns the role's one-shot Job within one poll. For a + # pipeline that was already RUNNING we skip this: its live event loop owns + # the respawn and a second driver thread would race it (#3244 review). + if pipeline_was_inactive: + _spawn_pipeline_run_thread(pipeline_id, store.repo_path, pipeline.run_epoch) + logger.info( + "restart_agent: relaunched driver thread for inactive pipeline", pipeline_id=pipeline_id, agent_role=agent_role, slice_id=slice_id, - error=str(count_err), + run_epoch=pipeline.run_epoch.isoformat() if pipeline.run_epoch else None, ) logger.info( diff --git a/orchestrator/tests/test_restart_agent.py b/orchestrator/tests/test_restart_agent.py index f8cf5cc86d..d2440ba528 100644 --- a/orchestrator/tests/test_restart_agent.py +++ b/orchestrator/tests/test_restart_agent.py @@ -326,6 +326,34 @@ def test_restart_counts_initialized_empty(self): assert hasattr(spawner, "_restart_counts") assert spawner._restart_counts == {} + def test_check_and_increment_restart_count_increments(self, spawner): + """check_and_increment_restart_count bumps and returns the new count (#3244).""" + assert spawner.check_and_increment_restart_count("issue-100", AgentRole.CODER) == 1 + assert spawner.check_and_increment_restart_count("issue-100", AgentRole.CODER) == 2 + assert spawner.get_restart_count("issue-100", "coder") == 2 + + def test_check_and_increment_restart_count_enforces_cap(self, spawner): + """Once the budget is exhausted, further calls raise instead of incrementing (#3244).""" + spawner._restart_counts[("issue-100", "coder", None)] = 2 + with pytest.raises(ContainerSpawnError, match="Restart limit"): + spawner.check_and_increment_restart_count( + "issue-100", AgentRole.CODER, max_restarts=2 + ) + # The rejected call must not burn an extra slot. + assert spawner.get_restart_count("issue-100", "coder") == 2 + + def test_check_and_increment_restart_count_slice_scoped(self, spawner): + """Slice-scoped budgets are independent of the pipeline-level bucket (#3244).""" + assert ( + spawner.check_and_increment_restart_count( + "issue-100", AgentRole.CODER, slice_id="slice-2" + ) + == 1 + ) + # Pipeline-level bucket untouched. + assert spawner.get_restart_count("issue-100", "coder") == 0 + assert spawner.get_restart_count("issue-100", "coder", slice_id="slice-2") == 1 + # --------------------------------------------------------------------------- # REST API endpoint tests (task-1-2) @@ -399,19 +427,22 @@ def client(app): class TestRestartAgentEndpoint: """Tests for POST //agents//restart endpoint.""" + @patch("routes.pipelines._spawn_pipeline_run_thread") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") def test_restart_agent_success( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client ): """Successful agent restart returns 200 and delegates respawn (#3164). After #3164 ``restart_agent`` no longer spawns a resident pod — it resets consensus + kills the live one-shot Job and the event loop respawns. So the response carries no ``container_id``; it - carries the ``respawn`` delegation marker. + carries the ``respawn`` delegation marker. The pipeline here is + RUNNING (live event loop), so the route must NOT relaunch a driver + thread — doing so would race the live loop (#3244 review). """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -419,11 +450,12 @@ def test_restart_agent_success( mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -439,6 +471,55 @@ def test_restart_agent_success( assert "container_id" not in data["data"] assert data["data"]["agent_role"] == "coder" assert data["data"]["respawn"] == "delegated to orchestrator event loop" + # restart_count reflects the just-incremented budget (#3244), not 0. + assert data["data"]["restart_count"] == 1 + # RUNNING pipeline: live event loop owns the respawn; no relaunch. + mock_spawn_thread.assert_not_called() + + @patch("routes.pipelines._spawn_pipeline_run_thread") + @patch("routes.pipelines.get_pipeline_state_lock") + @patch("routes.pipelines.get_container_spawner") + @patch("routes.pipelines._resolve_pipeline") + @patch("routes.pipelines.get_repo_path") + def test_restart_agent_rejected_when_budget_exhausted( + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client + ): + """Over-budget restart is rejected 429 without mutating state (#3244). + + The per-(pipeline, role, slice) restart cap that pre-#3164 lived in + ``restart_agent_job`` is re-enforced on the route. When the budget is + exhausted the route must reject loudly instead of resetting consensus + / flipping status and returning a misleading success — an unbounded + restart storm can actively prevent a converging phase from reaching + consensus. + """ + from kubernetes_spawner import KubernetesSpawnError + + mock_repo.return_value = "/repo" + mock_lock_fn.return_value = MagicMock() + pipeline = _make_pipeline_with_running_agent() + + mock_store = MagicMock() + mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") + mock_resolve.return_value = (mock_store, pipeline) + + mock_spawner = MagicMock() + mock_spawner.k8s.list_containers.return_value = [] + mock_spawner.check_and_increment_restart_count.side_effect = KubernetesSpawnError( + "Restart limit (2) exceeded for coder in pipeline issue-100 (restarted 2 times)" + ) + mock_spawner_fn.return_value = mock_spawner + + response = client.post( + "/api/v1/pipelines/issue-100/agents/coder/restart", + json={"reason": "Agent stalled"}, + ) + + assert response.status_code == 429 + # No destructive action: consensus/Jobs untouched, no relaunch. + mock_spawner.k8s.list_containers.assert_not_called() + mock_spawn_thread.assert_not_called() @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") @@ -491,12 +572,13 @@ def test_restart_agent_pipeline_not_running(self, mock_repo, mock_resolve, clien assert response.status_code == 409 + @patch("routes.pipelines._spawn_pipeline_run_thread") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") def test_restart_agent_cancelled_pipeline_resumes( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client ): """Restart succeeds on a cancelled pipeline and resets status to running. @@ -512,11 +594,12 @@ def test_restart_agent_cancelled_pipeline_resumes( mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -529,13 +612,16 @@ def test_restart_agent_cancelled_pipeline_resumes( assert pipeline.phases["implement"].status == PipelineStatus.RUNNING # Verify the CANCELLED -> RUNNING transition was persisted assert mock_store.update_pipeline.call_count >= 1 + # Dead event loop is restarted by relaunching the driver thread (#3244). + mock_spawn_thread.assert_called_once() + @patch("routes.pipelines._spawn_pipeline_run_thread") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") def test_restart_agent_failed_pipeline( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client ): """Restart succeeds on a failed pipeline and resets status to running.""" mock_repo.return_value = "/repo" @@ -546,11 +632,12 @@ def test_restart_agent_failed_pipeline( mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -561,6 +648,8 @@ def test_restart_agent_failed_pipeline( assert response.status_code == 200 assert pipeline.status == PipelineStatus.RUNNING assert pipeline.phases["implement"].status == PipelineStatus.RUNNING + # Dead event loop is restarted by relaunching the driver thread (#3244). + mock_spawn_thread.assert_called_once() @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @@ -585,7 +674,7 @@ def test_restart_tolerates_live_job_listing_failure( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -596,20 +685,24 @@ def test_restart_tolerates_live_job_listing_failure( assert response.status_code == 200 mock_spawner.restart_agent_container.assert_not_called() + @patch("routes.pipelines._spawn_pipeline_run_thread") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") def test_restart_failed_pipeline_transitions_to_running_no_revert( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client ): - """Restart of a FAILED pipeline transitions to RUNNING and stays there. + """Restart of a FAILED pipeline transitions to RUNNING and respawns. After #3164 there is no resident spawn that can fail, so the - early FAILED -> RUNNING transition is never reverted: the event - loop owns the respawn. Replaces the old - ``...spawner_failure_reverts_status_to_failed`` regression test - whose spawn-failure path no longer exists. + early FAILED -> RUNNING transition is never reverted. But a + FAILED pipeline's event loop and ``_run_pipeline`` driver thread + are already dead, so resetting consensus alone would leave the + pipeline RUNNING-but-idle (#3244 review). The route must relaunch + a fresh driver thread to restart the event loop — assert that + actually happens (the prior version of this test only checked the + status flip and never that a respawn was driven). """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -619,11 +712,12 @@ def test_restart_failed_pipeline_transitions_to_running_no_revert( mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -635,19 +729,26 @@ def test_restart_failed_pipeline_transitions_to_running_no_revert( assert pipeline.status == PipelineStatus.RUNNING assert pipeline.phases["implement"].status == PipelineStatus.RUNNING mock_spawner.restart_agent_container.assert_not_called() + # The dead event loop is restarted by relaunching the driver thread + # (mirrors restart_phase) — without this the respawn never happens. + mock_spawn_thread.assert_called_once() + assert mock_spawn_thread.call_args.args[0] == "issue-100" + @patch("routes.pipelines._spawn_pipeline_run_thread") @patch("routes.pipelines.get_pipeline_state_lock") @patch("routes.pipelines.get_container_spawner") @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") def test_restart_cancelled_pipeline_transitions_to_running_no_revert( - self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, client + self, mock_repo, mock_resolve, mock_spawner_fn, mock_lock_fn, mock_spawn_thread, client ): - """Restart of a CANCELLED pipeline transitions to RUNNING (no revert). + """Restart of a CANCELLED pipeline transitions to RUNNING and respawns. cancel_task(cleanup=false) leaves CANCELLED with state preserved; restart resumes it. With no resident spawn (#3164) there is no - failure path that would revert to FAILED. + failure path that would revert to FAILED. As with the FAILED path, + the dead event loop is restarted by relaunching the driver thread + (#3244 review). """ mock_repo.return_value = "/repo" mock_lock_fn.return_value = MagicMock() @@ -657,11 +758,12 @@ def test_restart_cancelled_pipeline_transitions_to_running_no_revert( mock_store = MagicMock() mock_store.load_pipeline.return_value = pipeline + mock_store.repo_path = Path("/repo") mock_resolve.return_value = (mock_store, pipeline) mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -673,6 +775,7 @@ def test_restart_cancelled_pipeline_transitions_to_running_no_revert( assert pipeline.status == PipelineStatus.RUNNING assert pipeline.phases["implement"].status == PipelineStatus.RUNNING mock_spawner.restart_agent_container.assert_not_called() + mock_spawn_thread.assert_called_once() @patch("routes.pipelines._resolve_pipeline") @patch("routes.pipelines.get_repo_path") @@ -725,7 +828,7 @@ def test_slice_id_query_param_scopes_live_job_lookup_and_count( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -743,9 +846,9 @@ def test_slice_id_query_param_scopes_live_job_lookup_and_count( LABEL_AGENT_ROLE: "coder", LABEL_SLICE_ID: "slice-2", } - # Reading the restart count after a slice-scoped restart MUST - # query the per-slice budget bucket (#2410). - get_count_call = mock_spawner.get_restart_count.call_args + # Enforcing/incrementing the restart budget after a slice-scoped + # restart MUST target the per-slice budget bucket (#2410/#3244). + get_count_call = mock_spawner.check_and_increment_restart_count.call_args assert get_count_call.kwargs.get("slice_id") == "slice-2" assert response.get_json()["data"]["slice_id"] == "slice-2" @@ -769,7 +872,7 @@ def test_slice_id_body_field_scopes_live_job_lookup_and_count( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -785,7 +888,7 @@ def test_slice_id_body_field_scopes_live_job_lookup_and_count( LABEL_AGENT_ROLE: "coder", LABEL_SLICE_ID: "slice-2", } - get_count_call = mock_spawner.get_restart_count.call_args + get_count_call = mock_spawner.check_and_increment_restart_count.call_args assert get_count_call.kwargs.get("slice_id") == "slice-2" @patch("routes.pipelines._resolve_pipeline") @@ -828,7 +931,7 @@ def test_no_slice_id_omits_slice_label_and_uses_none_count( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -844,7 +947,7 @@ def test_no_slice_id_omits_slice_label_and_uses_none_count( LABEL_AGENT_ROLE: "coder", } assert LABEL_SLICE_ID not in list_call.kwargs["labels"] - get_count_call = mock_spawner.get_restart_count.call_args + get_count_call = mock_spawner.check_and_increment_restart_count.call_args assert get_count_call.kwargs.get("slice_id") is None @patch("egg_contracts.loader.load_contract") @@ -928,7 +1031,7 @@ def test_known_slice_id_passes_validation( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -944,7 +1047,7 @@ def test_known_slice_id_passes_validation( # #3164: validation accepted, no resident spawn; the per-slice # count bucket is read. mock_spawner.restart_agent_container.assert_not_called() - assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-2" + assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-2" @patch("egg_contracts.loader.load_contract") @patch("routes.pipelines.get_container_spawner") @@ -1072,7 +1175,7 @@ def test_omitted_slice_id_derived_from_single_non_complete_record( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -1091,7 +1194,7 @@ def test_omitted_slice_id_derived_from_single_non_complete_record( assert ( mock_spawner.k8s.list_containers.call_args.kwargs["labels"][LABEL_SLICE_ID] == "slice-2" ) - assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-2" + assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-2" assert response.get_json()["data"]["slice_id"] == "slice-2" @patch("routes.pipelines.get_container_spawner") @@ -1189,7 +1292,7 @@ def test_explicit_slice_id_bypasses_derivation( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner with patch("egg_contracts.loader.load_contract") as mock_load_contract: @@ -1206,7 +1309,7 @@ def test_explicit_slice_id_bypasses_derivation( mock_spawner.restart_agent_container.assert_not_called() # The explicit slice_id wins over derivation: it scopes the count # bucket and the response. - assert mock_spawner.get_restart_count.call_args.kwargs.get("slice_id") == "slice-1" + assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-1" assert response.get_json()["data"]["slice_id"] == "slice-1" @@ -1292,7 +1395,7 @@ def test_slice_3_restart_does_not_mutate_slice_2_record( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner slice2_before = next( @@ -1377,7 +1480,7 @@ def test_slice_3_restart_with_no_existing_record_appends_with_slice_id( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner response = client.post( @@ -1749,7 +1852,7 @@ def test_live_job_deletion_failure_still_resets_consensus( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner mock_tracker = MagicMock() @@ -1793,7 +1896,7 @@ def test_restart_resets_consensus( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner mock_tracker = MagicMock() @@ -1839,7 +1942,7 @@ def test_successful_restart_calls_reset_agent( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner mock_hm = MagicMock() @@ -1886,7 +1989,7 @@ def test_successful_restart_refreshes_started_at( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.return_value = [] - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner with patch.dict( @@ -1942,7 +2045,7 @@ def test_live_job_deletion_failure_still_resets_health( mock_spawner = MagicMock() mock_spawner.k8s.list_containers.side_effect = RuntimeError("k8s API down") - mock_spawner.get_restart_count.return_value = 1 + mock_spawner.check_and_increment_restart_count.return_value = 1 mock_spawner_fn.return_value = mock_spawner mock_hm = MagicMock() From f2009c7fe018c483c7cdbaf3b03543b92f566b80 Mon Sep 17 00:00:00 2001 From: egg Date: Thu, 25 Jun 2026 16:34:34 +0000 Subject: [PATCH 3/3] Fix checks: apply automated formatting fixes --- orchestrator/tests/test_restart_agent.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/orchestrator/tests/test_restart_agent.py b/orchestrator/tests/test_restart_agent.py index d2440ba528..2290606c85 100644 --- a/orchestrator/tests/test_restart_agent.py +++ b/orchestrator/tests/test_restart_agent.py @@ -336,9 +336,7 @@ def test_check_and_increment_restart_count_enforces_cap(self, spawner): """Once the budget is exhausted, further calls raise instead of incrementing (#3244).""" spawner._restart_counts[("issue-100", "coder", None)] = 2 with pytest.raises(ContainerSpawnError, match="Restart limit"): - spawner.check_and_increment_restart_count( - "issue-100", AgentRole.CODER, max_restarts=2 - ) + spawner.check_and_increment_restart_count("issue-100", AgentRole.CODER, max_restarts=2) # The rejected call must not burn an extra slot. assert spawner.get_restart_count("issue-100", "coder") == 2 @@ -1047,7 +1045,10 @@ def test_known_slice_id_passes_validation( # #3164: validation accepted, no resident spawn; the per-slice # count bucket is read. mock_spawner.restart_agent_container.assert_not_called() - assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-2" + assert ( + mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") + == "slice-2" + ) @patch("egg_contracts.loader.load_contract") @patch("routes.pipelines.get_container_spawner") @@ -1194,7 +1195,10 @@ def test_omitted_slice_id_derived_from_single_non_complete_record( assert ( mock_spawner.k8s.list_containers.call_args.kwargs["labels"][LABEL_SLICE_ID] == "slice-2" ) - assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-2" + assert ( + mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") + == "slice-2" + ) assert response.get_json()["data"]["slice_id"] == "slice-2" @patch("routes.pipelines.get_container_spawner") @@ -1309,7 +1313,10 @@ def test_explicit_slice_id_bypasses_derivation( mock_spawner.restart_agent_container.assert_not_called() # The explicit slice_id wins over derivation: it scopes the count # bucket and the response. - assert mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") == "slice-1" + assert ( + mock_spawner.check_and_increment_restart_count.call_args.kwargs.get("slice_id") + == "slice-1" + ) assert response.get_json()["data"]["slice_id"] == "slice-1"