From bf12e32f29a93d85cd72727b750e4a3a32781c51 Mon Sep 17 00:00:00 2001 From: egg Date: Mon, 16 Mar 2026 04:51:58 +0000 Subject: [PATCH 1/5] Fix BRC consensus deadlocks (#1185) --- orchestrator/concurrent_executor.py | 17 +- orchestrator/peer_consensus.py | 73 +++++- orchestrator/review_graph.py | 11 + .../tests/test_peer_consensus_integration.py | 229 +++++++++++++++++- sandbox/.claude/commands/checker-mode.md | 67 ++--- 5 files changed, 360 insertions(+), 37 deletions(-) diff --git a/orchestrator/concurrent_executor.py b/orchestrator/concurrent_executor.py index f98db0e582..2c0dba582b 100644 --- a/orchestrator/concurrent_executor.py +++ b/orchestrator/concurrent_executor.py @@ -252,11 +252,12 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]: # Remove from consensus tracker = get_peer_consensus_tracker(self.pipeline.id) + crash_result = None if tracker: - result = tracker.handle_agent_crash(role) - if result.get("action") == "escalate": + crash_result = tracker.handle_agent_crash(role) + if crash_result.get("action") == "escalate": logger.warning( - "Agent crash requires escalation", role=role, reason=result.get("reason") + "Agent crash requires escalation", role=role, reason=crash_result.get("reason") ) emit_event( @@ -265,9 +266,16 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]: data={"role": role, "error": error}, ) + # Build context-aware HITL question + question = f"Agent '{role}' failed: {error}." + if crash_result and crash_result.get("blocking_producers"): + blocking = crash_result["blocking_producers"] + question += f" Reviewer had pending reviews for: {blocking}." + question += " How to proceed?" + # Create HITL decision decision = self.pipeline.add_decision( - question=f"Agent '{role}' failed: {error}. How to proceed?", + question=question, options=["Retry (respawn agent)", "Abort phase", "Continue without"], phase=self.pipeline.current_phase, ) @@ -284,6 +292,7 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]: "action": "hitl_decision", "decision_id": decision.id, "failed_role": role, + "crash_result": crash_result, } def _abort_phase(self, error: str, recent_failures: int) -> dict[str, Any]: diff --git a/orchestrator/peer_consensus.py b/orchestrator/peer_consensus.py index 33cf279260..c1f3ecd577 100644 --- a/orchestrator/peer_consensus.py +++ b/orchestrator/peer_consensus.py @@ -122,11 +122,27 @@ def _handle_propose_inner( self, agent_role: str, payload: dict[str, Any], + *, + _skip_ack_guard: bool = False, ) -> dict[str, Any]: """Inner propose logic. Caller MUST hold self._lock.""" if not self.graph.is_producer(agent_role): raise ValueError(f"{agent_role} is not a producer in this review graph") + # Guard: reject proposal when already fully ACKed and still in + # PROPOSED state (issue #1185). Skipped for handle_re_propose() which + # is always a legitimate path (after NACK or with new artifacts). + if ( + not _skip_ack_guard + and self.matrix.is_fully_acked(agent_role) + and self._producer_phases.get(agent_role) == ConsensusPhase.PROPOSED + ): + raise ValueError( + f"Producer {agent_role} is already fully ACKed " + f"(v{self.matrix.get_proposal_version(agent_role)}). " + f"Call confirmed instead of re-proposing." + ) + # Validate payload proposal = ProposalPayload(**payload) @@ -353,9 +369,14 @@ def handle_confirmed(self, agent_role: str) -> dict[str, Any]: # Check if agent can confirm if self.graph.is_producer(agent_role): if not self.matrix.is_fully_acked(agent_role): + blocking = self.matrix.get_blocking_edges(agent_role) + pending_reviewers = [e.reviewer_role for e in blocking] return { "status": "pending_acks", - "message": f"Producer {agent_role} waiting for reviewer re-ACKs", + "message": ( + f"Producer {agent_role} cannot confirm: not fully ACKed. " + f"Pending reviewers: {pending_reviewers}" + ), } self._producer_phases[agent_role] = ConsensusPhase.CONFIRMED @@ -423,8 +444,9 @@ def handle_re_propose( # The NACKing reviewer(s) always need to re-review # (their state is already NACKED in the matrix) - # Handle as a normal proposal while still holding the lock - result = self._handle_propose_inner(agent_role, payload) + # Handle as a normal proposal while still holding the lock. + # Skip the ACK guard — re-propose is always legitimate. + result = self._handle_propose_inner(agent_role, payload, _skip_ack_guard=True) result["invalidated_reviewers"] = invalidated return result @@ -437,16 +459,26 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]: pass if self.graph.is_reviewer(role): - # Reviewer crash: remove from graph review requirements - # Check if remaining ACKs suffice + # Reviewer crash: check impact on each assigned producer producers = self.graph.producers_for(role) sole_reviewer_for = [] + blocking_producers = [] for producer in producers: remaining_reviewers = [ r for r in self.graph.reviewers_for(producer) if r != role ] if not remaining_reviewers: sole_reviewer_for.append(producer) + else: + # Check if this reviewer had a pending (non-ACKed) review + entry = self.matrix.get_entry(role, producer) + latest_version = self.matrix.get_proposal_version(producer) + if ( + entry is None + or entry.state != ApprovalState.ACKED + or entry.version != latest_version + ): + blocking_producers.append(producer) if sole_reviewer_for: emit_event( @@ -463,11 +495,42 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]: "reason": f"Reviewer {role} crashed and was sole reviewer for {sole_reviewer_for}", } + if blocking_producers: + emit_event( + EventType.CONSENSUS_FAILURE, + self.pipeline_id, + data={ + "type": "reviewer_crash_pending", + "crashed_role": role, + "blocking_producers": blocking_producers, + }, + ) + return { + "action": "escalate", + "reason": f"Reviewer {role} crashed with pending reviews for {blocking_producers}", + "blocking_producers": blocking_producers, + } + # Remove from confirmed set self._confirmed.discard(role) return {"action": "continue", "crashed_role": role} + def excuse_reviewer(self, role: str) -> dict[str, Any]: + """Remove a reviewer from the review graph (HITL-gated). + + Called when a human decides to continue without a failed reviewer. + Removes all edges from this reviewer, allowing is_fully_acked() + to pass without their ACK. + """ + with self._lock: + producers = self.graph.producers_for(role) + for producer in producers: + self.graph.remove_edge(role, producer) + self._confirmed.discard(role) + self._reviewer_phases.pop(role, None) + return {"status": "excused", "role": role, "affected_producers": producers} + def is_timeout_handled(self) -> bool: """Check whether the BRC tracker has already handled the timeout.""" with self._lock: diff --git a/orchestrator/review_graph.py b/orchestrator/review_graph.py index d6e0a0afb5..1aa800d1d2 100644 --- a/orchestrator/review_graph.py +++ b/orchestrator/review_graph.py @@ -105,6 +105,17 @@ def get_edge(self, reviewer: str, producer: str) -> ReviewEdge | None: return e return None + def remove_edge(self, reviewer: str, producer: str) -> bool: + """Remove a review edge. Returns True if edge was found and removed.""" + for i, e in enumerate(self._edges): + if e.reviewer_role == reviewer and e.producer_role == producer: + self._edges.pop(i) + # Recalculate role sets + self._producer_roles = {e.producer_role for e in self._edges} + self._reviewer_roles = {e.reviewer_role for e in self._edges} + return True + return False + def all_roles(self) -> set[str]: """Get all roles participating in the graph.""" return self._producer_roles | self._reviewer_roles diff --git a/orchestrator/tests/test_peer_consensus_integration.py b/orchestrator/tests/test_peer_consensus_integration.py index 5d7c276c76..0aa3afcaac 100644 --- a/orchestrator/tests/test_peer_consensus_integration.py +++ b/orchestrator/tests/test_peer_consensus_integration.py @@ -1021,7 +1021,7 @@ def test_confirm_before_acked_returns_pending(self, tracker): # Coder tries to confirm before checker ACKs — should return pending, not raise result = tracker.handle_confirmed("coder") assert result["status"] == "pending_acks" - assert "waiting" in result["message"].lower() + assert "pending reviewers" in result["message"].lower() # Coder should NOT be in confirmed set assert "coder" not in tracker._confirmed @@ -1054,3 +1054,230 @@ def test_confirm_after_re_propose_invalidates_stale_acks(self, tracker): # Now coder tries to confirm — should get pending_acks result = tracker.handle_confirmed("coder") assert result["status"] == "pending_acks" + + +class TestReProposalGuard: + """Test that re-proposing when fully ACKed is rejected (issue #1185).""" + + def test_propose_rejected_when_fully_acked(self, tracker): + """Producer cannot re-propose after being fully ACKed.""" + # Coder proposes + tracker.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Both reviewers ACK + tracker.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + tracker.handle_ack( + "checker", "coder", {"artifact_references": ["src/auth.py"]} + ) + + # Verify fully ACKed + assert tracker.matrix.is_fully_acked("coder") + + # Re-proposing should raise ValueError + with pytest.raises(ValueError, match="already fully ACKed"): + tracker.handle_propose( + "coder", + {"summary": "v2", "artifacts": ["src/auth.py"]}, + ) + + def test_re_propose_allowed_after_nack(self, tracker): + """handle_re_propose is allowed after NACK (producer phase is WORKING).""" + tracker.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + tracker.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + # Checker NACKs instead of ACKing + tracker.handle_nack( + "checker", "coder", + {"artifact_references": ["src/auth.py"], "reason": "issues found"}, + ) + + # Re-proposing after NACK should work (producer phase is WORKING) + result = tracker.handle_re_propose( + "coder", + {"summary": "v2", "artifacts": ["src/auth.py"]}, + changed_artifacts=["src/auth.py"], + ) + assert result["status"] == "proposed" + + +class TestReviewerCrashPendingAck: + """Test reviewer crash with pending vs completed ACKs (issue #1185).""" + + def test_reviewer_crash_with_pending_ack_escalates(self): + """Non-sole reviewer crash with pending ACK escalates to HITL.""" + # Use implement graph: coder has reviewers reviewer_code, reviewer_contract, checker, tester + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Coder proposes + t.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Only reviewer_code ACKs; checker hasn't reviewed yet + t.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + + # Checker crashes with pending review -> should escalate + result = t.handle_agent_crash("checker") + assert result["action"] == "escalate" + assert "pending reviews" in result["reason"] + assert "coder" in result["blocking_producers"] + + def test_reviewer_crash_with_completed_ack_continues(self): + """Non-sole reviewer crash with completed ACK continues normally.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Coder proposes + t.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Both reviewers ACK + t.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + t.handle_ack( + "checker", "coder", {"artifact_references": ["src/auth.py"]} + ) + + # Checker crashes but already ACKed -> should continue + result = t.handle_agent_crash("checker") + assert result["action"] == "continue" + assert result["crashed_role"] == "checker" + + +class TestExcuseReviewer: + """Test excuse_reviewer unblocks consensus (issue #1185).""" + + def test_excuse_reviewer_unblocks_consensus(self): + """Excusing a dead reviewer allows is_fully_acked to pass.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Coder proposes + t.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Only reviewer_code ACKs + t.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + + # Not fully ACKed yet (checker hasn't reviewed) + assert not t.matrix.is_fully_acked("coder") + + # Excuse checker (simulating HITL "Continue without" decision) + result = t.excuse_reviewer("checker") + assert result["status"] == "excused" + assert "coder" in result["affected_producers"] + + # Now should be fully ACKed + assert t.matrix.is_fully_acked("coder") + + # Checker should no longer be a reviewer in the graph + assert not t.graph.is_reviewer("checker") + + +class TestRemoveEdge: + """Test ReviewGraph.remove_edge() (issue #1185).""" + + def test_remove_edge_success(self): + """remove_edge removes an existing edge and updates role sets.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + assert graph.is_reviewer("checker") + assert len(graph.reviewers_for("coder")) == 2 + + result = graph.remove_edge("checker", "coder") + assert result is True + assert len(graph.reviewers_for("coder")) == 1 + assert not graph.is_reviewer("checker") + + def test_remove_edge_not_found(self): + """remove_edge returns False for nonexistent edge.""" + graph = ReviewGraph( + [ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL)] + ) + result = graph.remove_edge("nonexistent", "coder") + assert result is False + + def test_remove_edge_preserves_other_edges(self): + """Removing one edge doesn't affect others.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("reviewer_code", "tester", ReviewCriticality.ADVISORY), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + graph.remove_edge("checker", "coder") + # reviewer_code should still review both coder and tester + assert graph.is_reviewer("reviewer_code") + assert "reviewer_code" in graph.reviewers_for("coder") + assert "reviewer_code" in graph.reviewers_for("tester") + + +class TestConfirmErrorListsPendingReviewers: + """Test improved error message for premature confirm (issue #1185).""" + + def test_confirm_error_lists_pending_reviewers(self, tracker): + """Premature confirm message lists which reviewers haven't ACKed.""" + tracker.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Only reviewer_code ACKs + tracker.handle_ack( + "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} + ) + + # Try to confirm — message should list checker as pending + result = tracker.handle_confirmed("coder") + assert result["status"] == "pending_acks" + assert "checker" in result["message"] + assert "Pending reviewers" in result["message"] diff --git a/sandbox/.claude/commands/checker-mode.md b/sandbox/.claude/commands/checker-mode.md index 5ea45b6245..12c8594ae0 100644 --- a/sandbox/.claude/commands/checker-mode.md +++ b/sandbox/.claude/commands/checker-mode.md @@ -89,36 +89,28 @@ cat > ".egg-state/agent-outputs/${IDENT}-checker-output.json" << 'EOF' EOF ``` -## Concurrent Mode +## Concurrent Mode (BRC Protocol) -When `EGG_CONCURRENT_MODE=true`, all agents start simultaneously. Your behavior changes: +When `EGG_CONCURRENT_MODE=true`, all agents start simultaneously. The checker is a **reviewer** for the coder in the BRC (Broadcast-Review-Converge) protocol. -### Startup — Wait for Coder +### Startup — Wait for Coder's Proposal -The coder hasn't committed anything yet. Signal BLOCKED and start polling: +The coder hasn't proposed yet. Start polling for their `CONSENSUS_PROPOSE` message: ```bash -egg-orch signal readiness --state BLOCKED --reason "Waiting for coder commits" -``` - -### While Waiting - -Poll for `PROGRESS` messages from the coder indicating new commits: - -```bash -egg-orch message poll +egg-orch message poll --wait 30 ``` While waiting, you can: - Review the plan to understand what checks will be relevant - Verify check infrastructure exists (Makefile targets, config files) -### When Coder Commits Land +### When Coder Proposes -When you receive a PROGRESS message or detect new commits: +When you receive a `CONSENSUS_PROPOSE` or `CONSENSUS_RE_REVIEW` message from the coder: -1. Signal `WORKING`: `egg-orch signal readiness --state WORKING --reason "Running checks on new commits"` -2. Run checks against the committed code +1. Pull the latest commits: `git pull origin` +2. Run checks against the committed code (lint, typecheck, tests) 3. Auto-fix what you can, commit fixes 4. Report unfixable issues to the coder: @@ -126,29 +118,50 @@ When you receive a PROGRESS message or detect new commits: egg-orch message send --to coder --type STATUS --subject "Check failures" --body "Type error in parser.py:45 — incompatible return type. Lint and tests pass." ``` -### Readiness +### ACK or NACK the Coder + +After running checks, issue your verdict: + +```bash +# All checks pass — ACK with attestation +egg-orch consensus ack coder \ + --files-reviewed "src/auth.py" "src/utils.py" \ + --summary "Lint clean, typecheck pass, all 42 tests pass. Auto-fixed 3 formatting issues." + +# Checks fail — NACK with specific, actionable reason +egg-orch consensus nack coder \ + --reason "Type error in parser.py:45 — incompatible return type. 2 test failures in test_auth.py." \ + --files-reviewed "src/parser.py" "src/auth.py" +``` + +**Attestation requirements for checker ACK/NACK:** +- Lint/type/test results (pass counts, failure details) +- Auto-fixes applied (files and description) +- Remaining warnings (count and severity) + +### Confirm When Done -Signal `READY` when all checks pass or unfixable issues are documented: +After ACKing all assigned producers: ```bash -egg-orch signal readiness --state READY --reason "All checks pass (auto-fixed 3 lint issues)" +egg-orch consensus confirmed ``` ### Stay-Alive Loop (CRITICAL) -**After signaling READY, do NOT exit.** Keep polling the message bus: +**After confirming, do NOT exit.** Keep polling for re-proposals: ```bash while true; do - egg-orch message poll - sleep "${EGG_MESSAGE_POLL_INTERVAL:-30}" + egg-orch message poll --wait "${EGG_MESSAGE_POLL_INTERVAL:-30}" done ``` -If the coder pushes more commits: -1. Signal `WORKING`: `egg-orch signal readiness --state WORKING --reason "Re-checking after new commits"` -2. Re-run checks -3. Signal `READY` again +If the coder re-proposes after a NACK: +1. You'll receive a `CONSENSUS_RE_REVIEW` message +2. Re-run checks on the updated code +3. ACK or NACK again +4. Confirm again after ACKing The orchestrator will stop your container when all agents reach consensus. From 168d499473f3d991ba93e6d806182c53fc88e090 Mon Sep 17 00:00:00 2001 From: egg Date: Mon, 16 Mar 2026 04:53:26 +0000 Subject: [PATCH 2/5] Fix checks: apply automated formatting fixes --- .../tests/test_peer_consensus_integration.py | 39 ++++++------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/orchestrator/tests/test_peer_consensus_integration.py b/orchestrator/tests/test_peer_consensus_integration.py index 0aa3afcaac..2a87f8a41f 100644 --- a/orchestrator/tests/test_peer_consensus_integration.py +++ b/orchestrator/tests/test_peer_consensus_integration.py @@ -1068,12 +1068,8 @@ def test_propose_rejected_when_fully_acked(self, tracker): ) # Both reviewers ACK - tracker.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) - tracker.handle_ack( - "checker", "coder", {"artifact_references": ["src/auth.py"]} - ) + tracker.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) + tracker.handle_ack("checker", "coder", {"artifact_references": ["src/auth.py"]}) # Verify fully ACKed assert tracker.matrix.is_fully_acked("coder") @@ -1091,12 +1087,11 @@ def test_re_propose_allowed_after_nack(self, tracker): "coder", {"summary": "v1", "artifacts": ["src/auth.py"]}, ) - tracker.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) + tracker.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) # Checker NACKs instead of ACKing tracker.handle_nack( - "checker", "coder", + "checker", + "coder", {"artifact_references": ["src/auth.py"], "reason": "issues found"}, ) @@ -1133,9 +1128,7 @@ def test_reviewer_crash_with_pending_ack_escalates(self): ) # Only reviewer_code ACKs; checker hasn't reviewed yet - t.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) + t.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) # Checker crashes with pending review -> should escalate result = t.handle_agent_crash("checker") @@ -1163,12 +1156,8 @@ def test_reviewer_crash_with_completed_ack_continues(self): ) # Both reviewers ACK - t.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) - t.handle_ack( - "checker", "coder", {"artifact_references": ["src/auth.py"]} - ) + t.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) + t.handle_ack("checker", "coder", {"artifact_references": ["src/auth.py"]}) # Checker crashes but already ACKed -> should continue result = t.handle_agent_crash("checker") @@ -1199,9 +1188,7 @@ def test_excuse_reviewer_unblocks_consensus(self): ) # Only reviewer_code ACKs - t.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) + t.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) # Not fully ACKed yet (checker hasn't reviewed) assert not t.matrix.is_fully_acked("coder") @@ -1239,9 +1226,7 @@ def test_remove_edge_success(self): def test_remove_edge_not_found(self): """remove_edge returns False for nonexistent edge.""" - graph = ReviewGraph( - [ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL)] - ) + graph = ReviewGraph([ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL)]) result = graph.remove_edge("nonexistent", "coder") assert result is False @@ -1272,9 +1257,7 @@ def test_confirm_error_lists_pending_reviewers(self, tracker): ) # Only reviewer_code ACKs - tracker.handle_ack( - "reviewer_code", "coder", {"artifact_references": ["src/auth.py"]} - ) + tracker.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) # Try to confirm — message should list checker as pending result = tracker.handle_confirmed("coder") From bf3a8eaaeba54910b915af6f65573b93d84c54d5 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 05:12:44 +0000 Subject: [PATCH 3/5] Fix review feedback: wire up excuse_reviewer, guard version 0 crash --- orchestrator/concurrent_executor.py | 3 + orchestrator/peer_consensus.py | 25 ++++---- orchestrator/routes/decisions.py | 27 ++++++++ orchestrator/tests/test_decisions_routes.py | 64 +++++++++++++++++++ .../tests/test_peer_consensus_integration.py | 50 ++++++++++++++- 5 files changed, 157 insertions(+), 12 deletions(-) diff --git a/orchestrator/concurrent_executor.py b/orchestrator/concurrent_executor.py index 2c0dba582b..6967ed06ed 100644 --- a/orchestrator/concurrent_executor.py +++ b/orchestrator/concurrent_executor.py @@ -279,6 +279,9 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]: options=["Retry (respawn agent)", "Abort phase", "Continue without"], phase=self.pipeline.current_phase, ) + # Store failed role in context so the resolution handler can call + # excuse_reviewer() when "Continue without" is selected. + decision.context = f"failed_role:{role}" logger.warning( "Single agent failure, HITL decision created", diff --git a/orchestrator/peer_consensus.py b/orchestrator/peer_consensus.py index c1f3ecd577..ea2a035431 100644 --- a/orchestrator/peer_consensus.py +++ b/orchestrator/peer_consensus.py @@ -453,6 +453,9 @@ def handle_re_propose( def handle_agent_crash(self, role: str) -> dict[str, Any]: """Handle an agent crash mid-protocol.""" with self._lock: + # Always clean up confirmed state on crash, even if we escalate + self._confirmed.discard(role) + if self.graph.is_producer(role): # Producer crash: proposal stands, reviewers continue # If reviewers NACK and producer can't respond, escalate @@ -470,15 +473,18 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]: if not remaining_reviewers: sole_reviewer_for.append(producer) else: - # Check if this reviewer had a pending (non-ACKed) review - entry = self.matrix.get_entry(role, producer) + # Check if this reviewer had a pending (non-ACKed) review. + # Skip producers that haven't proposed yet (version 0) — + # there's nothing to review so no blocking relationship. latest_version = self.matrix.get_proposal_version(producer) - if ( - entry is None - or entry.state != ApprovalState.ACKED - or entry.version != latest_version - ): - blocking_producers.append(producer) + if latest_version > 0: + entry = self.matrix.get_entry(role, producer) + if ( + entry is None + or entry.state != ApprovalState.ACKED + or entry.version != latest_version + ): + blocking_producers.append(producer) if sole_reviewer_for: emit_event( @@ -511,9 +517,6 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]: "blocking_producers": blocking_producers, } - # Remove from confirmed set - self._confirmed.discard(role) - return {"action": "continue", "crashed_role": role} def excuse_reviewer(self, role: str) -> dict[str, Any]: diff --git a/orchestrator/routes/decisions.py b/orchestrator/routes/decisions.py index a5e69a4907..8e387c77ee 100644 --- a/orchestrator/routes/decisions.py +++ b/orchestrator/routes/decisions.py @@ -37,6 +37,7 @@ def get_logger(name: str, **kwargs) -> logging.Logger: # type: ignore[misc] ) from events import EventType, emit_event from models import PipelinePhase +from peer_consensus import get_peer_consensus_tracker from state_store import InvalidPipelineIdError logger = get_logger("orchestrator.decisions") @@ -342,6 +343,32 @@ def resolve_decision(pipeline_id: str, decision_id: str) -> tuple[Response, int] exc_info=True, ) + # Handle "Continue without" resolution for failed reviewer decisions. + # The concurrent executor stores "failed_role:" in the decision + # context when a reviewer crashes. Excuse the reviewer so consensus + # can proceed without their ACK. + if decision.resolution == "Continue without" and decision.context.startswith( + "failed_role:" + ): + failed_role = decision.context.removeprefix("failed_role:") + tracker = get_peer_consensus_tracker(pipeline_id) + if tracker: + try: + excuse_result = tracker.excuse_reviewer(failed_role) + logger.info( + "Excused reviewer after 'Continue without' decision", + pipeline_id=pipeline_id, + failed_role=failed_role, + affected_producers=excuse_result.get("affected_producers"), + ) + except Exception: + logger.warning( + "Failed to excuse reviewer", + pipeline_id=pipeline_id, + failed_role=failed_role, + exc_info=True, + ) + return make_success_response( "Decision resolved", data={ diff --git a/orchestrator/tests/test_decisions_routes.py b/orchestrator/tests/test_decisions_routes.py index 805ffff812..ff73de359c 100644 --- a/orchestrator/tests/test_decisions_routes.py +++ b/orchestrator/tests/test_decisions_routes.py @@ -638,6 +638,70 @@ def test_resolve_succeeds_even_if_event_emission_fails( assert data["success"] is True +class TestContinueWithoutExcusesReviewer: + """Tests for 'Continue without' resolution calling excuse_reviewer.""" + + @patch("routes.decisions.get_peer_consensus_tracker") + @patch("routes.decisions.emit_event") + @patch("routes.decisions.get_repo_path") + @patch("routes.decisions.get_decision_queue") + def test_continue_without_calls_excuse_reviewer( + self, mock_get_queue, mock_repo, mock_emit, mock_get_tracker, client, tmp_path + ): + """Resolving with 'Continue without' calls excuse_reviewer for the failed role.""" + mock_repo.return_value = tmp_path + mock_queue = MagicMock() + resolved_decision = _make_decision( + status="resolved", + resolution="Continue without", + ) + resolved_decision.context = "failed_role:checker" + mock_queue.resolve_decision.return_value = resolved_decision + mock_get_queue.return_value = mock_queue + + mock_tracker = MagicMock() + mock_tracker.excuse_reviewer.return_value = { + "status": "excused", + "role": "checker", + "affected_producers": ["coder"], + } + mock_get_tracker.return_value = mock_tracker + + response = client.post( + "/api/v1/pipelines/test-pipeline/decisions/decision-1/resolve", + json={"resolution": "Continue without"}, + ) + + assert response.status_code == 200 + mock_tracker.excuse_reviewer.assert_called_once_with("checker") + + @patch("routes.decisions.get_peer_consensus_tracker") + @patch("routes.decisions.emit_event") + @patch("routes.decisions.get_repo_path") + @patch("routes.decisions.get_decision_queue") + def test_non_continue_without_does_not_call_excuse( + self, mock_get_queue, mock_repo, mock_emit, mock_get_tracker, client, tmp_path + ): + """Resolving with other options does not call excuse_reviewer.""" + mock_repo.return_value = tmp_path + mock_queue = MagicMock() + resolved_decision = _make_decision( + status="resolved", + resolution="Retry (respawn agent)", + ) + resolved_decision.context = "failed_role:checker" + mock_queue.resolve_decision.return_value = resolved_decision + mock_get_queue.return_value = mock_queue + + response = client.post( + "/api/v1/pipelines/test-pipeline/decisions/decision-1/resolve", + json={"resolution": "Retry (respawn agent)"}, + ) + + assert response.status_code == 200 + mock_get_tracker.assert_not_called() + + class TestCancelDecisionEndpoint: """Tests for POST cancel-decision endpoint.""" diff --git a/orchestrator/tests/test_peer_consensus_integration.py b/orchestrator/tests/test_peer_consensus_integration.py index 2a87f8a41f..98502dde27 100644 --- a/orchestrator/tests/test_peer_consensus_integration.py +++ b/orchestrator/tests/test_peer_consensus_integration.py @@ -1109,7 +1109,6 @@ class TestReviewerCrashPendingAck: def test_reviewer_crash_with_pending_ack_escalates(self): """Non-sole reviewer crash with pending ACK escalates to HITL.""" - # Use implement graph: coder has reviewers reviewer_code, reviewer_contract, checker, tester graph = ReviewGraph( [ ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), @@ -1136,6 +1135,24 @@ def test_reviewer_crash_with_pending_ack_escalates(self): assert "pending reviews" in result["reason"] assert "coder" in result["blocking_producers"] + def test_reviewer_crash_before_producer_proposes_continues(self): + """Reviewer crash when producer hasn't proposed yet should not escalate.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Checker crashes BEFORE coder proposes -> should continue, not escalate + result = t.handle_agent_crash("checker") + assert result["action"] == "continue" + assert result["crashed_role"] == "checker" + def test_reviewer_crash_with_completed_ack_continues(self): """Non-sole reviewer crash with completed ACK continues normally.""" graph = ReviewGraph( @@ -1164,6 +1181,37 @@ def test_reviewer_crash_with_completed_ack_continues(self): assert result["action"] == "continue" assert result["crashed_role"] == "checker" + def test_reviewer_crash_clears_confirmed_on_escalation(self): + """Reviewer crash clears confirmed state even when escalating.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Coder proposes + t.handle_propose( + "coder", + {"summary": "v1", "artifacts": ["src/auth.py"]}, + ) + + # Only reviewer_code ACKs; checker hasn't reviewed yet + t.handle_ack("reviewer_code", "coder", {"artifact_references": ["src/auth.py"]}) + + # Checker confirms (reviewer side — even though it's premature, for test setup) + # Manually add to confirmed set to simulate pre-crash state + t._confirmed.add("checker") + + # Checker crashes with pending review -> should escalate AND clear confirmed + result = t.handle_agent_crash("checker") + assert result["action"] == "escalate" + assert "checker" not in t._confirmed + class TestExcuseReviewer: """Test excuse_reviewer unblocks consensus (issue #1185).""" From fd685459a38f263e3c94e23bcbb0d612f15a78f6 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 05:28:24 +0000 Subject: [PATCH 4/5] Address review suggestions: validate excuse_reviewer role, include blocking_producers in sole_reviewer crash path --- orchestrator/peer_consensus.py | 11 +++- .../tests/test_peer_consensus_integration.py | 64 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/orchestrator/peer_consensus.py b/orchestrator/peer_consensus.py index ea2a035431..4da9d2e211 100644 --- a/orchestrator/peer_consensus.py +++ b/orchestrator/peer_consensus.py @@ -496,10 +496,15 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]: "unreviewed_producers": sole_reviewer_for, }, ) - return { + result: dict[str, Any] = { "action": "escalate", "reason": f"Reviewer {role} crashed and was sole reviewer for {sole_reviewer_for}", } + # Include blocking_producers so HITL gets complete info + # (reviewer may also have pending reviews for other producers) + if blocking_producers: + result["blocking_producers"] = blocking_producers + return result if blocking_producers: emit_event( @@ -525,8 +530,12 @@ def excuse_reviewer(self, role: str) -> dict[str, Any]: Called when a human decides to continue without a failed reviewer. Removes all edges from this reviewer, allowing is_fully_acked() to pass without their ACK. + + Raises ValueError if the role is not a reviewer in the graph. """ with self._lock: + if not self.graph.is_reviewer(role): + raise ValueError(f"Cannot excuse '{role}': not a reviewer in the review graph") producers = self.graph.producers_for(role) for producer in producers: self.graph.remove_edge(role, producer) diff --git a/orchestrator/tests/test_peer_consensus_integration.py b/orchestrator/tests/test_peer_consensus_integration.py index 98502dde27..f6549d9978 100644 --- a/orchestrator/tests/test_peer_consensus_integration.py +++ b/orchestrator/tests/test_peer_consensus_integration.py @@ -1312,3 +1312,67 @@ def test_confirm_error_lists_pending_reviewers(self, tracker): assert result["status"] == "pending_acks" assert "checker" in result["message"] assert "Pending reviewers" in result["message"] + + +class TestExcuseReviewerValidation: + """Test excuse_reviewer rejects non-reviewer roles.""" + + def test_excuse_non_reviewer_raises(self): + """excuse_reviewer raises ValueError for a non-reviewer role.""" + graph = ReviewGraph( + [ + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "coder", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + t.register_agent("checker") + + with pytest.raises(ValueError, match="not a reviewer"): + t.excuse_reviewer("coder") + + def test_excuse_unknown_role_raises(self): + """excuse_reviewer raises ValueError for an unknown role.""" + graph = ReviewGraph([ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL)]) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("reviewer_code") + + with pytest.raises(ValueError, match="not a reviewer"): + t.excuse_reviewer("nonexistent") + + +class TestSoleReviewerCrashIncludesBlockingProducers: + """Test sole_reviewer_for path includes blocking_producers when applicable.""" + + def test_sole_reviewer_crash_includes_blocking_producers(self): + """When reviewer is sole for some producers and blocking for others, + the crash result includes both pieces of information.""" + graph = ReviewGraph( + [ + # reviewer_code is sole reviewer for coder + ReviewEdge("reviewer_code", "coder", ReviewCriticality.CRITICAL), + # reviewer_code also reviews tester (with checker as backup) + ReviewEdge("reviewer_code", "tester", ReviewCriticality.CRITICAL), + ReviewEdge("checker", "tester", ReviewCriticality.CRITICAL), + ] + ) + t = PeerConsensusTracker("test-pipeline", graph, cooldown_seconds=0) + t.register_agent("coder") + t.register_agent("tester") + t.register_agent("reviewer_code") + t.register_agent("checker") + + # Both producers propose + t.handle_propose("coder", {"summary": "v1", "artifacts": ["src/a.py"]}) + t.handle_propose("tester", {"summary": "v1", "artifacts": ["test_a.py"]}) + + # reviewer_code crashes — sole for coder, pending for tester + result = t.handle_agent_crash("reviewer_code") + assert result["action"] == "escalate" + assert "sole reviewer" in result["reason"] + # Should include blocking_producers for tester + assert "blocking_producers" in result + assert "tester" in result["blocking_producers"] From bcc4059cc84784a529fe50b62af294ae4f8e86e1 Mon Sep 17 00:00:00 2001 From: "egg-reviewer[bot]" <261018737+egg-reviewer[bot]@users.noreply.github.com> Date: Mon, 16 Mar 2026 05:41:27 +0000 Subject: [PATCH 5/5] Address review nits: tighten test assertion, fix doc example --- docs/guides/agent-mode-design.md | 4 +++- orchestrator/tests/test_peer_consensus_integration.py | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/guides/agent-mode-design.md b/docs/guides/agent-mode-design.md index 7899f5658c..9368fce927 100644 --- a/docs/guides/agent-mode-design.md +++ b/docs/guides/agent-mode-design.md @@ -229,10 +229,12 @@ verdict = resp.json() **Right approach:** ```python # Delegate to a sandbox container running claude --print +from egg_agent import build_agent_command + spawner.spawn_agent_container( pipeline_id=f"{pipeline_id}-inspect", agent_role=AgentRole.INSPECTOR, - command=["claude", "--print", "--max-turns", "1", ...], + command=build_agent_command(max_turns=1, ...), ) ``` diff --git a/orchestrator/tests/test_peer_consensus_integration.py b/orchestrator/tests/test_peer_consensus_integration.py index f6549d9978..1051701f97 100644 --- a/orchestrator/tests/test_peer_consensus_integration.py +++ b/orchestrator/tests/test_peer_consensus_integration.py @@ -1373,6 +1373,5 @@ def test_sole_reviewer_crash_includes_blocking_producers(self): result = t.handle_agent_crash("reviewer_code") assert result["action"] == "escalate" assert "sole reviewer" in result["reason"] - # Should include blocking_producers for tester - assert "blocking_producers" in result - assert "tester" in result["blocking_producers"] + # Should include blocking_producers for tester (but NOT coder, which is in sole_reviewer_for) + assert result["blocking_producers"] == ["tester"]