Skip to content
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22,842 changes: 22,842 additions & 0 deletions .egg-state/brc-history/3064-implement-slice-5.json

Large diffs are not rendered by default.

18,928 changes: 18,928 additions & 0 deletions .egg-state/brc-history/3064-implement-slice-5.md

Large diffs are not rendered by default.

40 changes: 40 additions & 0 deletions orchestrator/concurrent_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,8 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
on_exhausted=self._teardown_exhausted_session,
)

# #3064 slice-5: convergence-stall notifier re-uses the same
# OVERSEER_ALERT surface wired for the supervisor.
loop = OrchestratorEventLoop(
tracker,
spawner_adapter,
Expand All @@ -504,6 +506,7 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
roles=make_role_list(roles),
job_supervisor=supervisor,
job_status_view=self._event_status_view,
convergence_stall_notifier=self._emit_supervision_alert,
)
self._event_loop = loop
loop.start()
Expand All @@ -514,8 +517,45 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
phase=phase,
roles=[r.value for r in roles],
)

# #3064 slice-5: set orchestrator mode on the health monitor and
# heartbeat coordinator so their tripwire/refresh behavior reflects
# the ownership mode (roles with no active Job are normal in
# orchestrator mode; gateway-session refresh via heartbeat fan-out
# is suppressed).
self._enable_orchestrator_mode_surfaces()

return loop

def _enable_orchestrator_mode_surfaces(self) -> None:
"""Propagate orchestrator mode to downstream surfaces.

In orchestrator mode:
- The health monitor suppresses alerts for roles with no active Job.
- The heartbeat coordinator suppresses gateway-session fan-out
(refresh happens at spawn time from slice-4 worktree re-attach).
- Absent-sender heartbeats between events trip nothing.

Best-effort: a missing health monitor or coordinator is tolerated
(unit tests often stand up only the component under test).
"""
try:
from health_monitor import get_health_monitor

hm = get_health_monitor()
if hm is not None:
hm.set_orchestrator_mode(True)
except Exception: # noqa: BLE001 — best-effort
pass

try:
from heartbeat import get_heartbeat_coordinator

hc = get_heartbeat_coordinator()
hc.set_orchestrator_mode(True)
except Exception: # noqa: BLE001 — best-effort
pass

def _build_event_spawn_params(
self, role: AgentRole
) -> tuple[list[str], str | None, str | None]:
Expand Down
208 changes: 204 additions & 4 deletions orchestrator/event_loop.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Orchestrator-owned BRC event loop (#3064 slice-2, TASK-2-1; slice-3, TASK-3-1).
"""Orchestrator-owned BRC event loop (#3064 slice-2, TASK-2-1; slice-3, TASK-3-1;
slice-5, TASK-5-1).

Under ``EGG_EVENT_LOOP_OWNER=orchestrator`` the orchestrator — not a
long-lived in-pod wait-loop — owns the BRC event loop. For every role it
Expand Down Expand Up @@ -27,6 +28,15 @@
applies backoff (streak*backoff capped), emits OVERSEER_ALERT on persistent
streaks, and resets on success. The wrapper template imports the constants
from the same source (``supervision_policy.py``).

**Slice-5 (#3064): convergence-stall detection (re-homed idle-budget)**

In orchestrator mode the in-pod ``check_idle_budget`` no longer runs.
Instead the event loop judges convergence stall from tracker timestamps:
when a role's derived actionable event (propose|ack|nack) has been pending
longer than ``EGG_BRC_IDLE_BUDGET_MIN`` with no BRC-bus activity, the loop
raises the same ``stuck-phase-transition`` anomaly the in-pod alert uses
today.
"""

from __future__ import annotations
Expand Down Expand Up @@ -125,6 +135,44 @@ def get_event_loop_poll_interval() -> float:
return val


# Convergence-stall idle budget (#3064 slice-5, re-homed from the in-pod
# ``check_idle_budget``). A role whose derived event has been pending
# longer than this many minutes without tracker-bus movement raises a
# ``stuck-phase-transition`` anomaly. Mirrors the wrapper's
# ``EVENT_PUMP_IDLE_BUDGET_MIN_DEFAULT = 30``.
_IDLE_BUDGET_MIN_DEFAULT = 30


def get_idle_budget_minutes() -> int:
"""Return the idle-budget threshold in minutes (default 30).

Reads ``EGG_BRC_IDLE_BUDGET_MIN``. Malformed or non-positive values
fall back to the default with a warning. The wrapper template reads
the same env var (``consensus_wrapper.py:151``); this function is the
orchestrator-side accessor for the re-homed convergence-stall check.
"""
raw = os.environ.get("EGG_BRC_IDLE_BUDGET_MIN", "").strip()
if not raw:
return _IDLE_BUDGET_MIN_DEFAULT
try:
val = int(raw)
except TypeError, ValueError:
logger.warning(
"EGG_BRC_IDLE_BUDGET_MIN=%r is not an integer; falling back to %d",
raw,
_IDLE_BUDGET_MIN_DEFAULT,
)
return _IDLE_BUDGET_MIN_DEFAULT
if val <= 0:
logger.warning(
"EGG_BRC_IDLE_BUDGET_MIN=%d must be > 0; falling back to %d",
val,
_IDLE_BUDGET_MIN_DEFAULT,
)
return _IDLE_BUDGET_MIN_DEFAULT
return val


def compute_dedupe_key(
pipeline_id: str,
slice_id: str | None,
Expand Down Expand Up @@ -488,6 +536,11 @@ class OrchestratorEventLoop:
effect as ``handler(action=, role=, payload=)``; ``clock`` is a monotonic
source the timing field reads. ``reconcile(live_dedupe_keys)`` seeds the
in-memory live set from live Job labels (the stateless-restart path).

Slice-5 (convergence-stall detection): inject ``convergence_stall_notifier``
to raise a ``stuck-phase-transition`` anomaly when a role has had a
pending actionable event (propose|ack|nack) for longer than
``EGG_BRC_IDLE_BUDGET_MIN`` minutes without BRC-bus activity.
"""

def __init__(
Expand All @@ -504,6 +557,7 @@ def __init__(
poll_interval: float | None = None,
job_supervisor: JobSupervisor | None = None,
job_status_view: Any | None = None,
convergence_stall_notifier: Callable[..., Any] | None = None,
) -> None:
self.tracker = tracker
self.spawner = spawner
Expand Down Expand Up @@ -531,6 +585,20 @@ def __init__(
# ``None`` (pure slice-2 mode / most unit tests) ⇒ no observation, so
# spawn behavior is byte-identical to slice-2.
self._job_status_view = job_status_view
# #3064 slice-5: convergence-stall notifier (raises
# ``stuck-phase-transition`` anomaly). When unset, stall detection
# is dormant — no alert emission (but the first-seen tracking still
# runs for diagnostic logging).
self._convergence_stall_notifier = convergence_stall_notifier
# Per-role first-seen wall-clock timestamp of the current stall
# window. {role: first_seen_wallclock_float} — set when a role is
# first observed with a pending actionable event after a period of
# tracker bus quiet; cleared when the role's derived action changes
# (no longer actionable) or the tracker bus moves.
self._stall_first_seen: dict[str, float] = {}
# Per-role sticky alert latch so the anomaly fires exactly once per
# stall episode (mirrors the in-pod ALERTED_AT_BUDGET flag).
self._stall_alerted: dict[str, bool] = {}

# ------------------------------------------------------------------
# Dedupe state
Expand Down Expand Up @@ -624,11 +692,21 @@ def poll_once(self, roles: list[str]) -> list[EventDecision]:
"""Run one derivation→action pass over ``roles``.

Observes finished Jobs first (slice-3 supervision), then derives the
next action per role. Returns a decision per role (role order). Never
raises: a per-role failure is logged and recorded as a no-op so one
bad role can't wedge the loop.
next action per role. Convergence-stall check runs after observation
so the tracker's bus timestamp is up to date. Returns a decision per
role (role order). Never raises: a per-role failure is logged and
recorded as a no-op so one bad role can't wedge the loop.
"""
self._observe_jobs()
try:
self._check_convergence_stall()
except Exception as exc: # noqa: BLE001 — never wedge the loop
logger.warning(
"event-loop convergence-stall check failed",
pipeline_id=self.pipeline_id,
slice_id=self.slice_id,
error=str(exc),
)
decisions: list[EventDecision] = []
for role in roles:
try:
Expand Down Expand Up @@ -715,6 +793,128 @@ def _handle_role(self, role: str) -> EventDecision:
)
return EventDecision(role=role, action=action, dedupe_key=key, spawned=True, timing=timing)

# ------------------------------------------------------------------
# Convergence-stall detection (#3064 slice-5, TASK-5-1)
# ------------------------------------------------------------------

def _check_convergence_stall(self) -> None:
"""Judge convergence stall from tracker timestamps.

Re-homed idle-budget check: for each role whose derived actionable
event (propose|ack|nack) has been pending longer than
``EGG_BRC_IDLE_BUDGET_MIN`` without BRC-bus activity, raises the
same ``stuck-phase-transition`` anomaly the in-pod
``check_idle_budget`` raises today (``consensus_wrapper.py:660-700``).

``confirm``/``complete``/``wait`` roles are never stalled — they're
making progress (agent-free side effects) or waiting on nothing.
A role with an in-flight Job (key in ``_live_keys``) is also not
stalled — the pod is handling the event.

The anomaly fires exactly once per stall episode per role (sticky
latch cleared when the BRC bus moves or the role's action changes).
Dormant when ``_convergence_stall_notifier`` is ``None``.
"""
if self._convergence_stall_notifier is None:
return

budget_min = get_idle_budget_minutes()
budget_sec = budget_min * 60
now = time.time()

# Latest BRC-bus activity from the tracker (proposals + ACK/NACK
# timestamps). ``None`` means no bus activity ever — in that case
# treat as "bus just started" so the first poll doesn't false-alert.
bus_ts = self.tracker.get_latest_progress_timestamp()
bus_timestamp: float = bus_ts.timestamp() if bus_ts is not None else now

# If the bus has moved within the budget window, reset ALL per-role
# stall state — the pipeline is clearly alive.
if now - bus_timestamp < budget_sec:
if self._stall_first_seen or self._stall_alerted:
logger.debug(
"Convergence-stall: BRC bus active %.0fs ago — resetting stall state",
now - bus_timestamp,
pipeline_id=self.pipeline_id,
slice_id=self.slice_id,
)
self._stall_first_seen.clear()
self._stall_alerted.clear()

for role in self._roles:
try:
action, _payload, _reason = _derive_next_action(self.tracker, role)
except Exception as exc: # noqa: BLE001 — per-role isolation
logger.debug(
"Convergence-stall: derivation failed for role",
pipeline_id=self.pipeline_id,
slice_id=self.slice_id,
role=role,
error=str(exc),
)
continue

# Agent-free (confirm/complete) and wait are never stalled.
if action in AGENT_FREE_ACTIONS or action == "wait" or action not in SPAWN_ACTIONS:
self._stall_first_seen.pop(role, None)
self._stall_alerted.pop(role, None)
continue

# If there's a live (in-flight) Job for this role, the event
# IS being handled — not stalled.
identity = event_identity(action, _payload)
key = compute_dedupe_key(
self.pipeline_id, self.slice_id, self.phase, role, action, identity
)
if key in self._live_keys:
self._stall_first_seen.pop(role, None)
continue

# Bus has moved since the budget started — reset per-role state
# (but already handled above for the all-roles reset, so just
# check the per-role flag for the already-alerted case).
if self._stall_alerted.get(role) and now - bus_timestamp < budget_sec:
self._stall_first_seen.pop(role, None)
self._stall_alerted.pop(role, None)
continue

# First time we observe this role with a pending actionable
# event — record the stall start.
if role not in self._stall_first_seen:
self._stall_first_seen[role] = now

elapsed = now - self._stall_first_seen[role]
if elapsed > budget_sec and not self._stall_alerted.get(role, False):
self._stall_alerted[role] = True
logger.warning(
"Convergence-stall detected: role=%s actionable event '%s' "
"pending %ds (budget %d min=%ds); raising stuck-phase-transition anomaly",
role,
action,
int(elapsed),
budget_min,
budget_sec,
pipeline_id=self.pipeline_id,
slice_id=self.slice_id,
phase=self.phase,
)
self._convergence_stall_notifier(
anomaly="stuck-phase-transition",
priority="high",
summary=(
f"orchestrator convergence stall: {role} {action} "
f"pending {int(elapsed)}s (budget {budget_min}m)"
),
detail=(
f"Event-loop for pipeline={self.pipeline_id} "
f"slice={self.slice_id} phase={self.phase} has "
f"derived action={action} for role={role} but the "
f"actionable event has been pending for {int(elapsed)}s "
f"without BRC-bus progress (budget={budget_min}m). "
f"No in-flight Job exists for this event."
),
)

# ------------------------------------------------------------------
# Background driver (production; not exercised by the unit contract)
# ------------------------------------------------------------------
Expand Down
Loading
Loading