Skip to content

fix(gateway): keep retryable platform reconnects queued #13197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
denhubr wants to merge 1 commit into from
+23 −19
Closed

fix(gateway): keep retryable platform reconnects queued #13197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions gateway/run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2401,10 +2401,9 @@ async def _platform_reconnect_watcher(self) -> None:
"""Background task that periodically retries connecting failed platforms.

Uses exponential backoff: 30s → 60s → 120s → 240s → 300s (cap).
Stops retrying a platform after 20 failed attempts or if the error
is non-retryable (e.g. bad auth token).
Retryable failures remain in the reconnect queue until they recover
or become non-retryable (e.g. bad auth token).
"""
_MAX_ATTEMPTS = 20
_BACKOFF_CAP = 300 # 5 minutes max between retries

await asyncio.sleep(10) # initial delay — let startup finish
Expand All @@ -2425,19 +2424,20 @@ async def _platform_reconnect_watcher(self) -> None:
if now < info["next_retry"]:
continue # not time yet

if info["attempts"] >= _MAX_ATTEMPTS:
logger.warning(
"Giving up reconnecting %s after %d attempts",
platform.value, info["attempts"],
)
del self._failed_platforms[platform]
continue

platform_config = info["config"]
attempt = info["attempts"] + 1
retry_delay = max(0, int(info["next_retry"] - now))
if attempt % 10 == 0:
logger.warning(
"Still retrying %s after %d failed attempt(s)",
platform.value,
info["attempts"],
)
logger.info(
"Reconnecting %s (attempt %d/%d)...",
platform.value, attempt, _MAX_ATTEMPTS,
"Reconnecting %s (attempt %d, retry delay %ds)...",
platform.value,
attempt,
retry_delay,
)

try:
Expand Down
16 changes: 10 additions & 6 deletions tests/gateway/test_platform_reconnect.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,20 +208,23 @@ async def fake_sleep(n):
assert runner._failed_platforms[Platform.TELEGRAM]["attempts"] == 2

@pytest.mark.asyncio
async def test_reconnect_gives_up_after_max_attempts(self):
"""After max attempts, platform should be removed from retry queue."""
async def test_reconnect_retryable_past_previous_max_attempts(self):
"""Retryable failures should remain queued even after many attempts."""
runner = _make_runner()

platform_config = PlatformConfig(enabled=True, token="test")
runner._failed_platforms[Platform.TELEGRAM] = {
"config": platform_config,
"attempts": 20, # At max
"attempts": 20,
"next_retry": time.monotonic() - 1,
}

fail_adapter = StubAdapter(
succeed=False, fatal_error="DNS failure", fatal_retryable=True
)
real_sleep = asyncio.sleep

with patch.object(runner, "_create_adapter") as mock_create:
with patch.object(runner, "_create_adapter", return_value=fail_adapter) as mock_create:
async def run_one_iteration():
runner._running = True
call_count = 0
Expand All @@ -238,8 +241,9 @@ async def fake_sleep(n):

await run_one_iteration()

assert Platform.TELEGRAM not in runner._failed_platforms
mock_create.assert_not_called() # Should give up without trying
assert Platform.TELEGRAM in runner._failed_platforms
assert runner._failed_platforms[Platform.TELEGRAM]["attempts"] == 21
mock_create.assert_called_once()

@pytest.mark.asyncio
async def test_reconnect_skips_when_not_time_yet(self):
Expand Down