Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -3449,6 +3449,7 @@ async def start(self) -> bool:
enabled_platform_count = 0
startup_nonretryable_errors: list[str] = []
startup_retryable_errors: list[str] = []
startup_degraded_reason: Optional[str] = None

# Initialize and connect each configured platform
for platform, platform_config in self.config.platforms.items():
Expand Down Expand Up @@ -3580,16 +3581,16 @@ async def start(self) -> bool:
return True
if enabled_platform_count > 0:
if startup_retryable_errors:
# At least one platform attempted a connection and failed —
# this is a real startup error that should block the gateway.
# All observed startup failures are retryable. Keep the
# process alive so the reconnect watcher can recover instead
# of relying on service-manager restart loops.
reason = "; ".join(startup_retryable_errors)
logger.error("Gateway failed to connect any configured messaging platform: %s", reason)
try:
from gateway.status import write_runtime_status
write_runtime_status(gateway_state="startup_failed", exit_reason=reason)
except Exception:
pass
return False
startup_degraded_reason = reason
logger.warning(
"Gateway failed to connect any messaging platform at startup; "
"keeping process alive for reconnect watcher: %s",
reason,
)
# All enabled platforms had no adapter (missing library or credentials).
# In fleet deployments the same config.yaml is shared across nodes that
# may only have credentials for a subset of platforms. Rather than
Expand All @@ -3609,7 +3610,10 @@ async def start(self) -> bool:
self._wire_teams_pipeline_runtime()

self._running = True
self._update_runtime_status("running")
if startup_degraded_reason and connected_count == 0 and self._failed_platforms:
self._update_runtime_status("degraded", startup_degraded_reason)
else:
self._update_runtime_status("running", None)

# Emit gateway:startup hook
hook_count = len(self.hooks.loaded_hooks)
Expand Down
11 changes: 8 additions & 3 deletions tests/gateway/test_runner_startup_failures.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ async def get_chat_info(self, chat_id):


@pytest.mark.asyncio
async def test_runner_returns_failure_for_retryable_startup_errors(monkeypatch, tmp_path):
async def test_runner_degrades_for_retryable_startup_errors(monkeypatch, tmp_path):
monkeypatch.setenv("HERMES_HOME", str(tmp_path))
config = GatewayConfig(
platforms={
Expand All @@ -75,16 +75,21 @@ async def test_runner_returns_failure_for_retryable_startup_errors(monkeypatch,
runner = GatewayRunner(config)

monkeypatch.setattr(runner, "_create_adapter", lambda platform, platform_config: _RetryableFailureAdapter())
monkeypatch.setattr(runner.hooks, "discover_and_load", lambda: None)
monkeypatch.setattr(runner.hooks, "emit", AsyncMock())

ok = await runner.start()

assert ok is False
assert ok is True
assert runner.should_exit_cleanly is False
state = read_runtime_status()
assert state["gateway_state"] == "startup_failed"
assert state["gateway_state"] == "degraded"
assert "temporary DNS resolution failure" in state["exit_reason"]
assert state["platforms"]["telegram"]["state"] == "retrying"
assert state["platforms"]["telegram"]["error_code"] == "telegram_connect_error"
assert Platform.TELEGRAM in runner._failed_platforms

await runner.stop()


@pytest.mark.asyncio
Expand Down