Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22,842 changes: 22,842 additions & 0 deletions .egg-state/brc-history/3064-implement-slice-5.json

Large diffs are not rendered by default.

18,928 changes: 18,928 additions & 0 deletions .egg-state/brc-history/3064-implement-slice-5.md

Large diffs are not rendered by default.

63 changes: 63 additions & 0 deletions orchestrator/concurrent_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -494,6 +494,8 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
on_exhausted=self._teardown_exhausted_session,
)

# #3064 slice-5: convergence-stall notifier re-uses the same
# OVERSEER_ALERT surface wired for the supervisor.
loop = OrchestratorEventLoop(
tracker,
spawner_adapter,
Expand All @@ -504,6 +506,8 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
roles=make_role_list(roles),
job_supervisor=supervisor,
job_status_view=self._event_status_view,
convergence_stall_notifier=self._emit_supervision_alert,
active_roles_notifier=self._publish_active_roles,
)
self._event_loop = loop
loop.start()
Expand All @@ -514,8 +518,67 @@ def _agent_free(*, action: str, role: str, payload: Any = None) -> None:
phase=phase,
roles=[r.value for r in roles],
)

# #3064 slice-5: set orchestrator mode on the health monitor and
# heartbeat coordinator so their tripwire/refresh behavior reflects
# the ownership mode (roles with no active Job are normal in
# orchestrator mode; gateway-session refresh via heartbeat fan-out
# is suppressed).
self._enable_orchestrator_mode_surfaces()

return loop

def _publish_active_roles(self, roles: set[str]) -> None:
"""Publish the event loop's live-Job role set to the health monitor.

Wired as the event loop's ``active_roles_notifier`` so the monitor's
``_active_jobs`` reflects which roles currently have an in-flight
one-shot Job on every poll tick. This is what makes orchestrator-mode
active-Job scoping (and silent-mid-event-pod coverage) actually take
effect in production — without it ``_active_jobs`` stays empty and
``_orchestrator_skip_tripwire`` suppresses every role.

Best-effort: a missing health monitor (unit tests stand up only the
component under test) is tolerated.
"""
try:
from health_monitor import get_health_monitor

hm = get_health_monitor()
if hm is not None:
hm.set_active_roles(roles)
except Exception: # noqa: BLE001 — best-effort
pass

def _enable_orchestrator_mode_surfaces(self) -> None:
"""Propagate orchestrator mode to downstream surfaces.

In orchestrator mode:
- The health monitor suppresses alerts for roles with no active Job.
- The heartbeat coordinator suppresses gateway-session fan-out
(refresh happens at spawn time from slice-4 worktree re-attach).
- Absent-sender heartbeats between events trip nothing.

Best-effort: a missing health monitor or coordinator is tolerated
(unit tests often stand up only the component under test).
"""
try:
from health_monitor import get_health_monitor

hm = get_health_monitor()
if hm is not None:
hm.set_orchestrator_mode(True)
except Exception: # noqa: BLE001 — best-effort
pass

try:
from heartbeat import get_heartbeat_coordinator

hc = get_heartbeat_coordinator()
hc.set_orchestrator_mode(True)
except Exception: # noqa: BLE001 — best-effort
pass

def owns_event_loop(self) -> bool:
"""True when the orchestrator-owned BRC event loop drives this phase.

Expand Down
11 changes: 10 additions & 1 deletion orchestrator/consensus_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,13 @@ def _event_loop_owner() -> str:
# default -- well above the WS7-observed 10-13 min legitimate-idle ceiling.
EVENT_PUMP_IDLE_BUDGET_MIN_DEFAULT = 30

# OVERSEER_ALERT anomaly name raised when the idle budget is exceeded. The
# orchestrator-side re-homed convergence-stall check (``event_loop.py``) must
# raise the SAME name so the overseer classifies the in-pod and orchestrator
# variants identically. Single-sourced here and interpolated into the wrapper
# template below; imported by ``event_loop`` for its OVERSEER_ALERT emission.
EVENT_PUMP_IDLE_BUDGET_ANOMALY = "stuck-phase-transition"

# Heartbeat cadence for the wrapper-owned background heartbeat emitter
# (#2908 task-2-2). Migrated from
# ``sandbox/egg_agent_tools/handlers/message.py:_WAIT_LOOP_HEARTBEAT_INTERVAL_SECS``
Expand Down Expand Up @@ -139,6 +146,7 @@ def _event_loop_owner() -> str:
# Placeholders interpolated by ``str.format``:
# {agent_command_prefix} -- ``python3 -m egg_agent --model X --max-turns N``
# {idle_budget_min_default}, {hb_interval_default}, {wait_timeout_default}
# {idle_budget_anomaly} -- OVERSEER_ALERT anomaly name on budget breach
_EVENT_PUMP_WRAPPER_TEMPLATE = r"""#!/bin/bash
set -uo pipefail

Expand Down Expand Up @@ -692,7 +700,7 @@ def _event_loop_owner() -> str:
" 2>/dev/null || echo "(snapshot unavailable)")
timeout 5 egg-orch overseer alert "${{EGG_PIPELINE_ID:-unknown}}" \
--role "${{EGG_AGENT_ROLE:-agent}}" \
--anomaly stuck-phase-transition \
--anomaly {idle_budget_anomaly} \
--priority "$priority" \
--summary "BRC event-pump idle for ${{idle}}s$summary_extra" \
--detail "Event-pump for role=${{EGG_AGENT_ROLE:-agent}} slice=${{EGG_SLICE_ID:-none}} has seen no actionable BRC event for ${{idle}}s (configured budget ${{IDLE_BUDGET_SECS}}s). The loop continues blocking; no FAILED transition is forced. BRC state: $brc_snapshot" \
Expand Down Expand Up @@ -1108,6 +1116,7 @@ def build_event_pump_wrapped_command(
script = _EVENT_PUMP_WRAPPER_TEMPLATE.format(
agent_command_prefix=agent_command_prefix,
idle_budget_min_default=idle_budget_min,
idle_budget_anomaly=EVENT_PUMP_IDLE_BUDGET_ANOMALY,
hb_interval_default=heartbeat_interval_secs,
wait_timeout_default=wait_timeout_secs,
spvr_backoff_factor=SUPERVISION_BACKOFF_FACTOR,
Expand Down
Loading
Loading