Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions gateway/platforms/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -4204,6 +4204,15 @@ async def disconnect(self) -> None:
await self._runner.cleanup()
self._runner = None
self._app = None
# Close the ResponseStore sqlite3 connection so the db + WAL file
# descriptors are released. Without this, every disconnect leaks 2
# fds, which adds up to gateway exhaustion on long-running deployments.
if self._response_store is not None:
try:
self._response_store.close()
except Exception:
pass
self._response_store = None
logger.info("[%s] API server stopped", self.name)

async def send(
Expand Down
30 changes: 30 additions & 0 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -6151,6 +6151,7 @@ async def _platform_reconnect_watcher(self) -> None:
platform.value, attempt,
)

adapter = None # ensure adapter is always bound for exception-path cleanup
try:
adapter = self._create_adapter(platform, platform_config)
if not adapter:
Expand Down Expand Up @@ -6200,6 +6201,7 @@ async def _platform_reconnect_watcher(self) -> None:
"Reconnect %s: non-retryable error (%s), removing from retry queue",
platform.value, adapter.fatal_error_message,
)
await self._dispose_failed_adapter(adapter, platform)
del self._failed_platforms[platform]
else:
self._update_platform_runtime_status(
Expand All @@ -6208,6 +6210,7 @@ async def _platform_reconnect_watcher(self) -> None:
error_code=adapter.fatal_error_code,
error_message=adapter.fatal_error_message or "failed to reconnect",
)
await self._dispose_failed_adapter(adapter, platform)
backoff = min(30 * (2 ** (attempt - 1)), _BACKOFF_CAP)
info["attempts"] = attempt
info["next_retry"] = time.monotonic() + backoff
Expand Down Expand Up @@ -6240,6 +6243,9 @@ async def _platform_reconnect_watcher(self) -> None:
# A raised exception during reconnect (connect timeout, DNS
# resolution failure, etc.) is inherently transient — keep
# retrying at the backoff cap rather than auto-pausing.
# Clean up the newly created adapter to release any fds it
# opened (e.g. APIServerAdapter sqlite3 db + WAL).
await self._dispose_failed_adapter(adapter, platform)

# Check every 10 seconds for platforms that need reconnection
for _ in range(10):
Expand Down Expand Up @@ -6606,6 +6612,30 @@ async def wait_for_shutdown(self) -> None:
"""Wait for shutdown signal."""
await self._shutdown_event.wait()

async def _dispose_failed_adapter(
self,
adapter: "Optional[BasePlatformAdapter]",
platform: "Platform",
) -> None:
"""Disconnect and clean up a newly created adapter that failed to connect.

Called in all three reconnect-loop failure paths so that any resources
the adapter opened during __init__ (e.g. APIServerAdapter sqlite3
ResponseStore connection — 2 fds per instance) are released before the
adapter reference is dropped. Without this, each failed reconnect
attempt leaks file descriptors until the process hits the OS fd limit
and becomes non-functional (~12-26 hours on a 300s backoff cycle).
"""
if adapter is None:
return
try:
await adapter.disconnect()
except Exception as exc:
logger.debug(
"Ignoring error while disposing failed %s adapter: %s",
platform.value, exc,
)

def _create_adapter(
self,
platform: Platform,
Expand Down
Loading