Skip to content
27 changes: 27 additions & 0 deletions gateway/gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -5543,6 +5543,33 @@ def session_delete_by_container(container_id: str) -> tuple[Response, int] | Res
return make_success("Session deleted")


@app.route("/api/v1/sessions/by-container/<container_id>/heartbeat", methods=["POST"])
@require_launcher_auth
def session_heartbeat_by_container(container_id: str) -> tuple[Response, int] | Response:
"""
Refresh a session's idle timer by container ID (orchestrator-only path).

Used by the orchestrator to keep agent sessions alive while their
container is heartbeating on the BRC bus but not making gateway
requests — without this, the idle pruner evicts the session after
EGG_SESSION_IDLE_TIMEOUT_MINUTES even though the agent is still
working (see #2068).

Auth: Bearer {launcher_secret}

Returns 404 if no session exists for the container. No per-session
rate limit because the launcher secret already gates access — only
the orchestrator can call this.
"""
session_manager = get_session_manager()
refreshed = session_manager.heartbeat_session_by_container(container_id)

if not refreshed:
return make_error("Session not found for container", status_code=404)

return make_success("Heartbeat recorded")


@app.route("/api/v1/sessions/<session_token>", methods=["GET"])
@require_launcher_auth
def session_get(session_token: str) -> tuple[Response, int] | Response:
Expand Down
33 changes: 33 additions & 0 deletions gateway/session_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -941,6 +941,39 @@ def delete_session_by_container(self, container_id: str) -> tuple[bool, threadin

return False, None

def heartbeat_session_by_container(self, container_id: str) -> bool:
"""Refresh a session's ``last_seen`` (and TTL) by container ID.

Used by the orchestrator to keep gateway sessions alive while an
agent is heartbeating through the BRC bus but not making gateway
requests (e.g. a producer in ``WAITING_FOR_EVENT`` during a long
review cycle). Without this, the idle pruner evicts the session
after ``DEFAULT_SESSION_IDLE_TIMEOUT_MINUTES`` even though the
agent is still working — see #2068.

Mirrors ``validate_session``'s in-flight pattern: the TTL extend
is in-memory only. We do **not** ``_save_to_disk()`` here — at
~1 fan-out per agent every 60s a per-call atomic file write
would dominate gateway disk I/O for no benefit (worst-case loss
on a gateway crash is the session ages out and is re-registered
on the next gateway op, which is the same recovery path
``validate_session`` already relies on). Disk persistence
happens at lifecycle events (registration, deletion, expiry).

Args:
container_id: Container ID whose session to refresh.

Returns:
True if a matching, non-expired session was refreshed; False
if no session exists for the container.
"""
with self._lock:
for session in self._sessions.values():
if session.container_id == container_id and not session.is_expired():
session.extend_ttl(self._ttl_hours)
return True
return False

def prune_expired_sessions(self) -> int:
"""
Remove all expired sessions.
Expand Down
40 changes: 40 additions & 0 deletions gateway/tests/test_gateway.py
Original file line number Diff line number Diff line change
Expand Up @@ -5118,6 +5118,46 @@ def test_session_delete_by_container_not_found(self, client, launcher_auth_heade
assert response.status_code == 404


class TestSessionHeartbeatByContainer:
"""Tests for POST /api/v1/sessions/by-container/<id>/heartbeat (#2068)."""

def test_refreshes_session(self, client, launcher_auth_headers):
mock_session_mgr = MagicMock()
mock_session_mgr.heartbeat_session_by_container.return_value = True

with patch.object(gateway, "get_session_manager", return_value=mock_session_mgr):
response = client.post(
"/api/v1/sessions/by-container/egg-agent-pipeline-1-coder/heartbeat",
headers=launcher_auth_headers,
)

assert response.status_code == 200
data = json.loads(response.data)
assert data["success"] is True
mock_session_mgr.heartbeat_session_by_container.assert_called_once_with(
"egg-agent-pipeline-1-coder"
)

def test_returns_404_when_session_missing(self, client, launcher_auth_headers):
mock_session_mgr = MagicMock()
mock_session_mgr.heartbeat_session_by_container.return_value = False

with patch.object(gateway, "get_session_manager", return_value=mock_session_mgr):
response = client.post(
"/api/v1/sessions/by-container/nonexistent/heartbeat",
headers=launcher_auth_headers,
)

assert response.status_code == 404

def test_requires_launcher_auth(self, client):
"""Endpoint must reject requests without the launcher secret."""
response = client.post(
"/api/v1/sessions/by-container/egg-agent-pipeline-1-coder/heartbeat",
)
assert response.status_code == 401


class TestBranchIsolation:
"""Tests for branch isolation enforcement in pipeline worktree sessions.

Expand Down
78 changes: 78 additions & 0 deletions gateway/tests/test_session_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -779,6 +779,84 @@ def test_delete_clears_token_cache(self, manager):
assert not result.valid


class TestHeartbeatByContainer:
"""Tests for heartbeat_session_by_container (#2068)."""

@pytest.fixture
def manager(self, tmp_path):
return SessionManager(persistence_file=tmp_path / "sessions.json")

def test_heartbeat_refreshes_last_seen(self, manager):
"""A heartbeat call advances last_seen so the idle pruner spares the session."""
_token, session = manager.register_session(
container_id="egg-agent-pipeline-1-coder",
container_ip="172.18.0.5",
mode="private",
)

# Backdate last_seen past the idle timeout window.
session.last_seen = datetime.now(UTC) - timedelta(minutes=120)

refreshed = manager.heartbeat_session_by_container("egg-agent-pipeline-1-coder")
assert refreshed is True

# Session is now fresh -- not pruned by a 60-minute idle sweep.
assert manager.prune_idle_sessions(idle_timeout_minutes=60) == 0
assert manager.get_session_by_container("egg-agent-pipeline-1-coder") is not None

def test_heartbeat_unknown_container(self, manager):
"""Heartbeat for an unknown container returns False without raising."""
assert manager.heartbeat_session_by_container("not-a-real-container") is False

def test_heartbeat_extends_ttl(self, manager):
"""Heartbeat advances expires_at to ``now + ttl`` (mirrors validate_session).

The 23h54m floor (rather than e.g. 23h59m) is a deliberate 6-minute
slack to absorb test-execution jitter between ``register_session``
and the post-heartbeat ``datetime.now(UTC)`` call: a stalled
runner could in principle take a few seconds between the two,
and the assertion only needs to prove ``extend_ttl`` actually
fired (not that the clock was perfectly still). The strict
``>`` against ``backdated_expiry`` (``now + 23.5h``) is what
proves the heartbeat moved the deadline forward — the 23h54m
floor is the *upper-bound* sanity check that it landed near
``now + 24h`` rather than being clamped lower.
"""
_token, session = manager.register_session(
container_id="egg-agent-pipeline-1-coder",
container_ip="172.18.0.5",
mode="private",
)
# Backdate both ``last_seen`` and ``expires_at`` 30 minutes into
# the past. ``extend_ttl`` should pull ``expires_at`` forward to
# roughly ``now + 24h`` — strictly greater than the backdated
# value of ``now + 23.5h``.
backdated_last_seen = datetime.now(UTC) - timedelta(minutes=30)
session.last_seen = backdated_last_seen
session.expires_at = backdated_last_seen + timedelta(hours=24)
backdated_expiry = session.expires_at

manager.heartbeat_session_by_container("egg-agent-pipeline-1-coder")
refreshed_session = manager.get_session_by_container("egg-agent-pipeline-1-coder")
assert refreshed_session is not None
# Strict ``>`` proves the heartbeat moved ``expires_at`` forward;
# 23h54m floor proves it landed near ``now + 24h`` (sanity check
# — see docstring for the 6-minute slack rationale).
assert refreshed_session.expires_at > backdated_expiry
assert refreshed_session.expires_at > datetime.now(UTC) + timedelta(hours=23, minutes=54)

def test_heartbeat_ignores_expired_session(self, manager):
"""Already-expired sessions are not silently revived."""
_token, session = manager.register_session(
container_id="egg-agent-pipeline-1-coder",
container_ip="172.18.0.5",
mode="private",
)
session.expires_at = datetime.now(UTC) - timedelta(minutes=1)

assert manager.heartbeat_session_by_container("egg-agent-pipeline-1-coder") is False


class TestSessionModes:
"""Tests for session mode handling."""

Expand Down
30 changes: 30 additions & 0 deletions orchestrator/gateway_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -573,6 +573,36 @@ def delete_session_by_container(self, container_id: str) -> bool:
)
return False

def heartbeat_session_by_container(self, container_id: str) -> bool:
"""Refresh a session's idle timer by container ID.

Requires launcher secret authentication. Used to keep gateway
sessions alive while an agent is heartbeating on the BRC bus but
not making gateway requests — see #2068.

Args:
container_id: Container ID whose session to refresh.

Returns:
True if the session was refreshed; False if there is no
matching session or the gateway request failed. Best-effort
— callers should not fail on a False return.
"""
try:
result = self._make_request(
f"/api/v1/sessions/by-container/{quote(container_id, safe='')}/heartbeat",
method="POST",
use_launcher_auth=True,
)
return result.get("success", False)
except GatewayError as e:
logger.warning(
"Failed to heartbeat session by container",
container_id=container_id[:12] if len(container_id) >= 12 else container_id,
error=str(e),
)
return False

def create_worktrees(
self,
container_id: str,
Expand Down
63 changes: 63 additions & 0 deletions orchestrator/routes/messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,19 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]:
# budget (review NB1, issue #1897). States in
# ``_DEDUP_EXEMPT_HEARTBEAT_STATES`` skip this check; see the
# constant's docstring for the rationale.
#
# Note: the gateway-session fan-out below runs *after* dedup but
# *before* rate-limit. Dedup'd heartbeats still fan out so an agent
# stuck in a single state (e.g. ``WORKING`` through a slow
# ``make test``) keeps its gateway session alive even when its BRC
# state hasn't changed. Rate-limited heartbeats do not fan out: by
# definition the agent already got plenty of refreshes in the last
# minute, and a hot-looping agent shouldn't amplify into the
# gateway.
if state not in _DEDUP_EXEMPT_HEARTBEAT_STATES and coordinator.is_duplicate(
pipeline_id, from_role, state, waiting_on
):
_refresh_gateway_session(pipeline_id, from_role)
return _make_success(
"HEARTBEAT deduped (unchanged state)",
data={"deduped": True},
Expand Down Expand Up @@ -549,6 +559,14 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]:
resp.headers["Retry-After"] = str(retry_after)
return resp, 429

# Refresh the agent's gateway session liveness (#2068). Runs after
# dedup and rate-limit gates: every accepted-or-deduped heartbeat
# fans out (dedup'd path above), but rate-limited ones do not.
# Best-effort: the gateway may be unreachable (tests, dev runs
# without a gateway) and a missing session is a 404; never fail the
# heartbeat on this path.
_refresh_gateway_session(pipeline_id, from_role)

# Emit as a normal HEARTBEAT message on the bus so downstream
# consumers (HealthMonitor, overseer, UI) see it.
metadata = {"state": state}
Expand Down Expand Up @@ -589,6 +607,51 @@ def post_heartbeat(pipeline_id: str) -> tuple[Response, int]:
)


def _refresh_gateway_session(pipeline_id: str, from_role: str) -> None:
"""Best-effort POST to the gateway so the BRC heartbeat counts as session liveness.

Container-id normalization: k8s names are RFC-1123 labels (no
underscores), so ``kubernetes_spawner.JOB_NAME_FORMAT`` is filled
with ``agent_role.value.replace("_", "-")``. ``from_role`` arrives
from ``EGG_AGENT_ROLE`` which is the underscore form, so we mirror
the same normalization here — otherwise roles like
``reviewer_refine`` build a container_id that never matches the
registered session and the gateway returns 404. See
``orchestrator/kubernetes_spawner.py:370-375`` for the reference
pattern.

Trust model: ``from_role`` is taken at face value from the request
body and is **not** correlated against the calling container's
session. This matches the existing message-bus trust model — any
agent in any container can already post messages claiming to be
another role — but with this fan-out a misbehaving agent can keep
a sibling's gateway session alive past the idle timeout. Tracked
as a follow-up; spoofing here doesn't grant any new capability,
only extends an existing session's lifetime.
"""
try:
try:
from gateway_client import get_gateway_client
except ImportError: # pragma: no cover
from ..gateway_client import (
get_gateway_client, # type: ignore[no-redef,import-not-found]
)

# Mirror kubernetes_spawner.JOB_NAME_FORMAT's role normalization
# — k8s labels disallow underscores, so the registered
# container_id uses hyphens.
normalized_role = from_role.replace("_", "-")
container_id = f"egg-agent-{pipeline_id}-{normalized_role}"
get_gateway_client().heartbeat_session_by_container(container_id)
except Exception as exc: # pragma: no cover - logging only
logger.warning(
"Gateway session heartbeat fan-out failed",
pipeline_id=pipeline_id,
from_role=from_role,
error=str(exc),
)


@messages_bp.route("/<pipeline_id>/messages/status", methods=["GET"])
def message_status(pipeline_id: str) -> tuple[Response, int]:
"""Get message bus status for a pipeline."""
Expand Down
30 changes: 30 additions & 0 deletions orchestrator/tests/test_gateway_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,10 @@ def do_POST(self):
self._handle_git_fetch(data)
elif self.path == "/api/v1/gh/pr/create":
self._handle_pr_create(data)
elif self.path.startswith("/api/v1/sessions/by-container/") and self.path.endswith(
"/heartbeat"
):
self._handle_heartbeat_by_container()
else:
self._send_error(404, "Not found")

Expand Down Expand Up @@ -169,6 +173,20 @@ def _handle_delete_by_container(self):

self._send_json({"success": True})

def _handle_heartbeat_by_container(self):
"""Handle session heartbeat by container ID (POST .../by-container/<id>/heartbeat)."""
auth_header = self.headers.get("Authorization", "")
if not auth_header.startswith("Bearer ") or auth_header[7:] != "test-secret":
self._send_error(401, "Unauthorized")
return

# Path: /api/v1/sessions/by-container/<id>/heartbeat
container_id = self.path.split("/")[-2]
if container_id == "missing":
self._send_json({"success": False, "message": "Session not found"}, status=404)
return
self._send_json({"success": True})

def _handle_worktree_create(self, data):
"""Handle worktree creation (POST /api/v1/worktree/create)."""
auth_header = self.headers.get("Authorization", "")
Expand Down Expand Up @@ -487,6 +505,18 @@ def test_delete_session_by_container(self, gateway_client, mock_gateway_server):
result = gateway_client.delete_session_by_container("container-123")
assert result is True

def test_heartbeat_session_by_container(self, gateway_client, mock_gateway_server):
"""Heartbeat-by-container returns True for an active session."""
result = gateway_client.heartbeat_session_by_container("egg-agent-pipe-1-coder")
assert result is True

def test_heartbeat_session_by_container_missing_returns_false(
self, gateway_client, mock_gateway_server
):
"""Heartbeat-by-container swallows 404 and returns False (best-effort path)."""
result = gateway_client.heartbeat_session_by_container("missing")
assert result is False


class TestSecurityBoundaryValidation:
"""Tests for security boundary validation."""
Expand Down
Loading
Loading