Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions orchestrator/concurrent_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -252,11 +252,12 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]:

# Remove from consensus
tracker = get_peer_consensus_tracker(self.pipeline.id)
crash_result = None
if tracker:
result = tracker.handle_agent_crash(role)
if result.get("action") == "escalate":
crash_result = tracker.handle_agent_crash(role)
if crash_result.get("action") == "escalate":
logger.warning(
"Agent crash requires escalation", role=role, reason=result.get("reason")
"Agent crash requires escalation", role=role, reason=crash_result.get("reason")
)

emit_event(
Expand All @@ -265,12 +266,22 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]:
data={"role": role, "error": error},
)

# Build context-aware HITL question
question = f"Agent '{role}' failed: {error}."
if crash_result and crash_result.get("blocking_producers"):
blocking = crash_result["blocking_producers"]
question += f" Reviewer had pending reviews for: {blocking}."
question += " How to proceed?"

# Create HITL decision
decision = self.pipeline.add_decision(
question=f"Agent '{role}' failed: {error}. How to proceed?",
question=question,
options=["Retry (respawn agent)", "Abort phase", "Continue without"],
phase=self.pipeline.current_phase,
)
# Store failed role in context so the resolution handler can call
# excuse_reviewer() when "Continue without" is selected.
decision.context = f"failed_role:{role}"

logger.warning(
"Single agent failure, HITL decision created",
Expand All @@ -284,6 +295,7 @@ def _handle_single_failure(self, role: str, error: str) -> dict[str, Any]:
"action": "hitl_decision",
"decision_id": decision.id,
"failed_role": role,
"crash_result": crash_result,
}

def _abort_phase(self, error: str, recent_failures: int) -> dict[str, Any]:
Expand Down
91 changes: 83 additions & 8 deletions orchestrator/peer_consensus.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,11 +122,27 @@ def _handle_propose_inner(
self,
agent_role: str,
payload: dict[str, Any],
*,
_skip_ack_guard: bool = False,
) -> dict[str, Any]:
"""Inner propose logic. Caller MUST hold self._lock."""
if not self.graph.is_producer(agent_role):
raise ValueError(f"{agent_role} is not a producer in this review graph")

# Guard: reject proposal when already fully ACKed and still in
# PROPOSED state (issue #1185). Skipped for handle_re_propose() which
# is always a legitimate path (after NACK or with new artifacts).
if (
not _skip_ack_guard
and self.matrix.is_fully_acked(agent_role)
and self._producer_phases.get(agent_role) == ConsensusPhase.PROPOSED
):
raise ValueError(
f"Producer {agent_role} is already fully ACKed "
f"(v{self.matrix.get_proposal_version(agent_role)}). "
f"Call confirmed instead of re-proposing."
)

# Validate payload
proposal = ProposalPayload(**payload)

Expand Down Expand Up @@ -353,9 +369,14 @@ def handle_confirmed(self, agent_role: str) -> dict[str, Any]:
# Check if agent can confirm
if self.graph.is_producer(agent_role):
if not self.matrix.is_fully_acked(agent_role):
blocking = self.matrix.get_blocking_edges(agent_role)
pending_reviewers = [e.reviewer_role for e in blocking]
return {
"status": "pending_acks",
"message": f"Producer {agent_role} waiting for reviewer re-ACKs",
"message": (
f"Producer {agent_role} cannot confirm: not fully ACKed. "
f"Pending reviewers: {pending_reviewers}"
),
}
self._producer_phases[agent_role] = ConsensusPhase.CONFIRMED

Expand Down Expand Up @@ -423,30 +444,47 @@ def handle_re_propose(
# The NACKing reviewer(s) always need to re-review
# (their state is already NACKED in the matrix)

# Handle as a normal proposal while still holding the lock
result = self._handle_propose_inner(agent_role, payload)
# Handle as a normal proposal while still holding the lock.
# Skip the ACK guard — re-propose is always legitimate.
result = self._handle_propose_inner(agent_role, payload, _skip_ack_guard=True)
result["invalidated_reviewers"] = invalidated
return result

def handle_agent_crash(self, role: str) -> dict[str, Any]:
"""Handle an agent crash mid-protocol."""
with self._lock:
# Always clean up confirmed state on crash, even if we escalate
self._confirmed.discard(role)

if self.graph.is_producer(role):
# Producer crash: proposal stands, reviewers continue
# If reviewers NACK and producer can't respond, escalate
pass

if self.graph.is_reviewer(role):
# Reviewer crash: remove from graph review requirements
# Check if remaining ACKs suffice
# Reviewer crash: check impact on each assigned producer
producers = self.graph.producers_for(role)
sole_reviewer_for = []
blocking_producers = []
for producer in producers:
remaining_reviewers = [
r for r in self.graph.reviewers_for(producer) if r != role
]
if not remaining_reviewers:
sole_reviewer_for.append(producer)
else:
# Check if this reviewer had a pending (non-ACKed) review.
# Skip producers that haven't proposed yet (version 0) —
# there's nothing to review so no blocking relationship.
latest_version = self.matrix.get_proposal_version(producer)
if latest_version > 0:
entry = self.matrix.get_entry(role, producer)
if (
entry is None
or entry.state != ApprovalState.ACKED
or entry.version != latest_version
):
blocking_producers.append(producer)

if sole_reviewer_for:
emit_event(
Expand All @@ -458,16 +496,53 @@ def handle_agent_crash(self, role: str) -> dict[str, Any]:
"unreviewed_producers": sole_reviewer_for,
},
)
return {
result: dict[str, Any] = {
"action": "escalate",
"reason": f"Reviewer {role} crashed and was sole reviewer for {sole_reviewer_for}",
}
# Include blocking_producers so HITL gets complete info
# (reviewer may also have pending reviews for other producers)
if blocking_producers:
result["blocking_producers"] = blocking_producers
return result

# Remove from confirmed set
self._confirmed.discard(role)
if blocking_producers:
emit_event(
EventType.CONSENSUS_FAILURE,
self.pipeline_id,
data={
"type": "reviewer_crash_pending",
"crashed_role": role,
"blocking_producers": blocking_producers,
},
)
return {
"action": "escalate",
"reason": f"Reviewer {role} crashed with pending reviews for {blocking_producers}",
"blocking_producers": blocking_producers,
}

return {"action": "continue", "crashed_role": role}

def excuse_reviewer(self, role: str) -> dict[str, Any]:
"""Remove a reviewer from the review graph (HITL-gated).

Called when a human decides to continue without a failed reviewer.
Removes all edges from this reviewer, allowing is_fully_acked()
to pass without their ACK.

Raises ValueError if the role is not a reviewer in the graph.
"""
with self._lock:
if not self.graph.is_reviewer(role):
raise ValueError(f"Cannot excuse '{role}': not a reviewer in the review graph")
producers = self.graph.producers_for(role)
for producer in producers:
self.graph.remove_edge(role, producer)
self._confirmed.discard(role)
self._reviewer_phases.pop(role, None)
return {"status": "excused", "role": role, "affected_producers": producers}

def is_timeout_handled(self) -> bool:
"""Check whether the BRC tracker has already handled the timeout."""
with self._lock:
Expand Down
11 changes: 11 additions & 0 deletions orchestrator/review_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,17 @@ def get_edge(self, reviewer: str, producer: str) -> ReviewEdge | None:
return e
return None

def remove_edge(self, reviewer: str, producer: str) -> bool:
"""Remove a review edge. Returns True if edge was found and removed."""
for i, e in enumerate(self._edges):
if e.reviewer_role == reviewer and e.producer_role == producer:
self._edges.pop(i)
# Recalculate role sets
self._producer_roles = {e.producer_role for e in self._edges}
self._reviewer_roles = {e.reviewer_role for e in self._edges}
return True
return False

def all_roles(self) -> set[str]:
"""Get all roles participating in the graph."""
return self._producer_roles | self._reviewer_roles
Expand Down
27 changes: 27 additions & 0 deletions orchestrator/routes/decisions.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ def get_logger(name: str, **kwargs) -> logging.Logger: # type: ignore[misc]
)
from events import EventType, emit_event
from models import PipelinePhase
from peer_consensus import get_peer_consensus_tracker
from state_store import InvalidPipelineIdError

logger = get_logger("orchestrator.decisions")
Expand Down Expand Up @@ -342,6 +343,32 @@ def resolve_decision(pipeline_id: str, decision_id: str) -> tuple[Response, int]
exc_info=True,
)

# Handle "Continue without" resolution for failed reviewer decisions.
# The concurrent executor stores "failed_role:<role>" in the decision
# context when a reviewer crashes. Excuse the reviewer so consensus
# can proceed without their ACK.
if decision.resolution == "Continue without" and decision.context.startswith(
"failed_role:"
):
failed_role = decision.context.removeprefix("failed_role:")
tracker = get_peer_consensus_tracker(pipeline_id)
if tracker:
try:
excuse_result = tracker.excuse_reviewer(failed_role)
logger.info(
"Excused reviewer after 'Continue without' decision",
pipeline_id=pipeline_id,
failed_role=failed_role,
affected_producers=excuse_result.get("affected_producers"),
)
except Exception:
logger.warning(
"Failed to excuse reviewer",
pipeline_id=pipeline_id,
failed_role=failed_role,
exc_info=True,
)

return make_success_response(
"Decision resolved",
data={
Expand Down
64 changes: 64 additions & 0 deletions orchestrator/tests/test_decisions_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,70 @@ def test_resolve_succeeds_even_if_event_emission_fails(
assert data["success"] is True


class TestContinueWithoutExcusesReviewer:
"""Tests for 'Continue without' resolution calling excuse_reviewer."""

@patch("routes.decisions.get_peer_consensus_tracker")
@patch("routes.decisions.emit_event")
@patch("routes.decisions.get_repo_path")
@patch("routes.decisions.get_decision_queue")
def test_continue_without_calls_excuse_reviewer(
self, mock_get_queue, mock_repo, mock_emit, mock_get_tracker, client, tmp_path
):
"""Resolving with 'Continue without' calls excuse_reviewer for the failed role."""
mock_repo.return_value = tmp_path
mock_queue = MagicMock()
resolved_decision = _make_decision(
status="resolved",
resolution="Continue without",
)
resolved_decision.context = "failed_role:checker"
mock_queue.resolve_decision.return_value = resolved_decision
mock_get_queue.return_value = mock_queue

mock_tracker = MagicMock()
mock_tracker.excuse_reviewer.return_value = {
"status": "excused",
"role": "checker",
"affected_producers": ["coder"],
}
mock_get_tracker.return_value = mock_tracker

response = client.post(
"/api/v1/pipelines/test-pipeline/decisions/decision-1/resolve",
json={"resolution": "Continue without"},
)

assert response.status_code == 200
mock_tracker.excuse_reviewer.assert_called_once_with("checker")

@patch("routes.decisions.get_peer_consensus_tracker")
@patch("routes.decisions.emit_event")
@patch("routes.decisions.get_repo_path")
@patch("routes.decisions.get_decision_queue")
def test_non_continue_without_does_not_call_excuse(
self, mock_get_queue, mock_repo, mock_emit, mock_get_tracker, client, tmp_path
):
"""Resolving with other options does not call excuse_reviewer."""
mock_repo.return_value = tmp_path
mock_queue = MagicMock()
resolved_decision = _make_decision(
status="resolved",
resolution="Retry (respawn agent)",
)
resolved_decision.context = "failed_role:checker"
mock_queue.resolve_decision.return_value = resolved_decision
mock_get_queue.return_value = mock_queue

response = client.post(
"/api/v1/pipelines/test-pipeline/decisions/decision-1/resolve",
json={"resolution": "Retry (respawn agent)"},
)

assert response.status_code == 200
mock_get_tracker.assert_not_called()


class TestCancelDecisionEndpoint:
"""Tests for POST cancel-decision endpoint."""

Expand Down
Loading
Loading