Skip to content
Merged
6 changes: 6 additions & 0 deletions orchestrator/events.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,12 @@ class EventType(StrEnum):

# Progress monitoring
PROGRESS_EMITTED = "progress.emitted"
# Container-side activity (e.g. successful mcp__task__add_commit) that
# demonstrates an agent is alive even when no HEARTBEAT message-bus
# traffic exists. Consumed by HealthMonitor to suppress
# heartbeat/progress stall alerts against agents legitimately blocked
# in long tool calls. See issue #2190.
CONTAINER_ACTIVITY = "container.activity"

# System events — health check framework (see health_checks/runner.py)
HEALTH_CHECK = "system.health_check"
Expand Down
98 changes: 98 additions & 0 deletions orchestrator/health_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,13 @@ class AgentState:
agent_id: str
last_heartbeat: float = field(default_factory=time.time)
last_progress: float = field(default_factory=time.time)
# Wall-clock timestamp of the most recent CONTAINER_ACTIVITY event for
# this agent (e.g. successful git commit registration). Used to
# OR-suppress heartbeat/progress stall alerts against agents that are
# legitimately blocked in long tool calls but still making real
# progress (issue #2190). Defaults to 0.0 — never seen — so a freshly
# spawned agent's silence is governed by the heartbeat anchor alone.
last_activity: float = 0.0
error_counts: dict[str, int] = field(default_factory=lambda: defaultdict(int))
message_timestamps: list[float] = field(default_factory=list)
heartbeat_escalated: bool = False
Expand Down Expand Up @@ -131,6 +138,7 @@ def __init__(
self._event_bus.subscribe(EventType.ERROR, self._on_error)
self._event_bus.subscribe(EventType.CONTAINER_STOPPED, self._on_container_stopped)
self._event_bus.subscribe(EventType.MESSAGE_SENT, self._on_message_sent)
self._event_bus.subscribe(EventType.CONTAINER_ACTIVITY, self._on_container_activity)

# -----------------------------------------------------------------
# Callback registration
Expand Down Expand Up @@ -211,6 +219,45 @@ def _get_heartbeat_threshold(self) -> int:
return self._config.orchestrator_implement_heartbeat_timeout_seconds
return self._config.orchestrator_heartbeat_timeout_seconds

def _has_recent_activity(self, agent_id: str, now: float) -> tuple[bool, str | None]:
"""Return (defer, reason) for the focal-agent activity gate (#2190).

Suppresses ``heartbeat_timeout`` / ``progress_stall`` alerts against
an agent that is demonstrably alive — i.e. has emitted a
:class:`EventType.CONTAINER_ACTIVITY` event within
``orchestrator_activity_quiet_seconds`` — even if no
bus-level ``HEARTBEAT`` has arrived. The repro from #2190 was a
coder mid-pytest with multi-minute blocking ``TaskOutput`` calls,
committing along the way; the bus saw silence but the agent was
making real progress.

Complements #2242's :func:`_has_recent_peer_progress` (peer
signals; self-excluded). The two gates are OR'd: either the
focal agent's own activity OR a peer's recent progress is
sufficient to defer.

Returns ``(False, None)`` when the agent has never emitted an
activity event (``last_activity == 0.0``) so a freshly spawned
but truly silent agent still escalates on the heartbeat anchor.

``orchestrator_activity_quiet_seconds <= 0`` disables the gate.
"""
threshold = self._config.orchestrator_activity_quiet_seconds
if threshold <= 0:
return False, None

with self._lock:
agent = self._agents.get(agent_id)
last_activity = agent.last_activity if agent is not None else 0.0

if last_activity <= 0.0:
return False, None

age = now - last_activity
if 0 <= age < threshold:
return True, f"container activity {int(age)}s ago"
return False, None

def _is_brc_idle(self, agent_id: str) -> bool:
"""Check if an agent is idle waiting for BRC upstream producers.

Expand Down Expand Up @@ -308,6 +355,26 @@ def _on_progress(self, event: Event) -> None:
if new_state != "blocked" or new_blocker != old_blocker:
agent.infra_error_escalated = False

def _on_container_activity(self, event: Event) -> None:
"""Handle CONTAINER_ACTIVITY event — record focal-agent activity (#2190).

Activity events fire on demonstrable signs of life that don't go
through the bus-level HEARTBEAT path: today, successful commit
registrations from the gateway commit observer. Used by
:func:`_has_recent_activity` to suppress heartbeat/progress
alerts against agents legitimately blocked in long tool calls.
"""
if event.pipeline_id != self._pipeline_id:
return

agent_id = event.data.get("agent_id") or event.data.get("agent_role")
if not agent_id:
return

with self._lock:
agent = self._get_or_create_agent(agent_id)
agent.last_activity = time.time()

def _on_error(self, event: Event) -> None:
"""Handle ERROR event — track repeated identical errors."""
if event.pipeline_id != self._pipeline_id:
Expand Down Expand Up @@ -482,6 +549,22 @@ def check_heartbeats(self) -> list[dict[str, Any]]:
if self._is_brc_idle(agent_id):
continue

# Focal-agent activity gate (#2190): defer if the agent has
# had recent CONTAINER_ACTIVITY (e.g. successful commit) — it
# is alive, just not emitting bus-level HEARTBEAT (typical of
# long-blocking tool calls). Don't set heartbeat_escalated;
# the next poll re-checks.
defer, gate_reason = self._has_recent_activity(agent_id, now)
if defer:
logger.info(
"Heartbeat alert deferred by activity gate",
pipeline_id=self._pipeline_id,
agent_id=agent_id,
elapsed_seconds=int(elapsed),
reason=gate_reason,
)
continue

action = {
"action": "escalate",
"agent_id": agent_id,
Expand Down Expand Up @@ -565,6 +648,20 @@ def check_progress(self) -> list[dict[str, Any]]:
if self._is_brc_idle(agent_id):
continue

# Focal-agent activity gate (#2190): same as in
# check_heartbeats — defer when the agent has had recent
# CONTAINER_ACTIVITY but no bus-level progress event.
defer, gate_reason = self._has_recent_activity(agent_id, now)
if defer:
logger.info(
"Progress alert deferred by activity gate",
pipeline_id=self._pipeline_id,
agent_id=agent_id,
elapsed_seconds=int(elapsed),
reason=gate_reason,
)
continue

action = {
"action": "escalate",
"agent_id": agent_id,
Expand Down Expand Up @@ -685,6 +782,7 @@ def stop(self, pipeline_id: str | None = None) -> None:
self._event_bus.unsubscribe(EventType.ERROR, self._on_error)
self._event_bus.unsubscribe(EventType.CONTAINER_STOPPED, self._on_container_stopped)
self._event_bus.unsubscribe(EventType.MESSAGE_SENT, self._on_message_sent)
self._event_bus.unsubscribe(EventType.CONTAINER_ACTIVITY, self._on_container_activity)

def _check_infra_errors(self) -> list[dict[str, Any]]:
"""Detect blocked progress events with infrastructure error keywords.
Expand Down
9 changes: 9 additions & 0 deletions orchestrator/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,15 @@ class PipelineConfig(BaseModel):
orchestrator_message_rate_limit: int = Field(
default=20, ge=1, description="Max messages per minute before auto-throttle"
)
orchestrator_activity_quiet_seconds: int = Field(
default=120,
ge=1,
description=(
"Seconds since the last CONTAINER_ACTIVITY event below which an "
"agent is considered alive — suppresses heartbeat/progress stall "
"alerts even when bus-level HEARTBEATs are absent (issue #2190)."
),
)
overseer_poll_interval_seconds: int = Field(
default=30, ge=5, description="Overseer polling interval in seconds"
)
Expand Down
16 changes: 16 additions & 0 deletions orchestrator/overseer/decision_maker.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,22 @@ async def decide_corrective_action(
' "restart_phase" - Restart all agents in the current phase (requires HITL approval)\n'
' "issue" - File a diagnostic GitHub issue\n'
' "slack" - Send urgent Slack notification\n\n'
"Recommendation ladder for stall / silent-agent classifications "
"(issue #2190): inspecting agent state must come before any "
"container-restart recommendation. An apparently-silent agent is "
"often mid-tool-call (e.g. a multi-minute pytest). Restarting "
"would destroy in-flight commits.\n"
" - First response: `nudge` or `redirect` whose message body "
'leads with "Inspect container logs via '
"`mcp__egg__get_container_logs(task_id=…, agent_role=…)` before "
'taking destructive action."\n'
" - Do NOT recommend `restart_agent` for a first stall alert. "
"Reserve `restart_agent` for follow-up alerts that fire after a "
"log inspection has confirmed the agent is genuinely inactive "
"(no recent commits, no pushes, no tool-call results), or for "
"infrastructure errors classified separately.\n"
" - Never embed `egg-orch container restart <id>` as a first-line "
"operator action in the message body.\n\n"
"Respond with ONLY a JSON object (no markdown fences) with these keys:\n"
' "action": one of the actions above\n'
' "message": string describing the action or message to send\n'
Expand Down
46 changes: 46 additions & 0 deletions orchestrator/routes/commit_authorship.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,36 @@ def get_logger(name: str, **kwargs: Any): # type: ignore[misc]

logger = get_logger("orchestrator.commit_authorship")


def _publish_container_activity(pipeline_id: str | None, role: str, kind: str) -> None:
"""Best-effort publish of a CONTAINER_ACTIVITY event.

The event lets HealthMonitor suppress heartbeat/progress stall alerts
against agents that are demonstrably alive (mid-commit) but not
emitting bus-level HEARTBEATs. See issue #2190.

Failure to publish must not affect the registration response.
"""
if not isinstance(pipeline_id, str) or not pipeline_id.strip():
return
try:
try:
from events import Event, EventType, get_event_bus # type: ignore[import-not-found]
except ImportError:
from ..events import Event, EventType, get_event_bus # type: ignore[no-redef]

get_event_bus().publish(
Event(
event_type=EventType.CONTAINER_ACTIVITY,
pipeline_id=pipeline_id,
data={"agent_role": role, "kind": kind},
source="commit_authorship",
)
)
except Exception as exc: # pragma: no cover - defensive
logger.warning("container_activity_publish_failed", error=str(exc))


commit_authorship_bp = Blueprint(
"commit_authorship", __name__, url_prefix="/api/v1/commit-authorship"
)
Expand Down Expand Up @@ -165,6 +195,8 @@ def register_commit() -> tuple[Response, int] | Response:
)
return _json_error("Failed to register commit authorship", 500)

_publish_container_activity(pipeline_id, role.strip().lower(), "git_commit")

return (
jsonify(
{
Expand Down Expand Up @@ -206,6 +238,7 @@ def register_bulk() -> tuple[Response, int] | Response:
return _json_error("Commit-authorship store unavailable", 500)

results: list[dict[str, Any]] = []
activity_seen: set[tuple[str, str]] = set()
for item in items:
if not isinstance(item, dict):
results.append({"success": False, "message": "Item must be an object"})
Expand Down Expand Up @@ -251,6 +284,19 @@ def register_bulk() -> tuple[Response, int] | Response:
}
)

item_pipeline = item.get("pipeline_id")
item_role = item.get("role", "")
if (
isinstance(item_pipeline, str)
and item_pipeline
and isinstance(item_role, str)
and item_role
):
key = (item_pipeline, item_role.strip().lower())
if key not in activity_seen:
activity_seen.add(key)
_publish_container_activity(item_pipeline, key[1], "git_commit")

return jsonify({"success": True, "results": results}), 200


Expand Down
51 changes: 51 additions & 0 deletions orchestrator/tests/test_commit_authorship_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,57 @@ def register(self, *_a, **_kw):
# Generic message — no internal details.
assert "simulated backing-store failure" not in body["message"]

def test_register_publishes_container_activity(self, client, monkeypatch):
"""A successful registration publishes a CONTAINER_ACTIVITY event so
HealthMonitor can suppress heartbeat/progress alerts against an
agent that is demonstrably alive (#2190)."""
import routes.commit_authorship as route_mod # type: ignore[import-not-found]
from events import EventBus, EventType # type: ignore[import-not-found]

bus = EventBus(async_delivery=False)
monkeypatch.setattr(
route_mod, "_publish_container_activity", route_mod._publish_container_activity
)
# Reroute get_event_bus inside the route module's helper.
import events as events_mod # type: ignore[import-not-found]

monkeypatch.setattr(events_mod, "get_event_bus", lambda: bus)

captured: list = []
bus.subscribe(EventType.CONTAINER_ACTIVITY, lambda e: captured.append(e))

response = client.post(
"/api/v1/commit-authorship/register",
data=json.dumps({"sha": _VALID_SHA, "role": "coder", "pipeline_id": "issue-1882"}),
content_type="application/json",
)
assert response.status_code == 200
assert len(captured) == 1
evt = captured[0]
assert evt.pipeline_id == "issue-1882"
assert evt.data["agent_role"] == "coder"
assert evt.data["kind"] == "git_commit"

def test_register_without_pipeline_id_does_not_publish(self, client, monkeypatch):
"""Orphan registrations (no pipeline_id) do not publish CONTAINER_ACTIVITY —
the event has no pipeline scope to attach to."""
import events as events_mod # type: ignore[import-not-found]
from events import EventBus, EventType # type: ignore[import-not-found]

bus = EventBus(async_delivery=False)
monkeypatch.setattr(events_mod, "get_event_bus", lambda: bus)

captured: list = []
bus.subscribe(EventType.CONTAINER_ACTIVITY, lambda e: captured.append(e))

response = client.post(
"/api/v1/commit-authorship/register",
data=json.dumps({"sha": _VALID_SHA, "role": "coder"}),
content_type="application/json",
)
assert response.status_code == 200
assert captured == []


# ---------------------------------------------------------------------------
# /lookup — batch attribution
Expand Down
Loading
Loading