Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 94 additions & 0 deletions gateway/error_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
"""Transient-error classification + loop handler extracted from gateway/run.py (#54962).

Fifth slice of the gateway god-file unpacking: the transient network
error classifier and the asyncio loop-level exception handler that uses
it. The classifier is a pure function (walk the cause chain, match known
transient exception class names); the handler wires it into the event
loop's safety net so a transient peer error can never kill the gateway
process (#31066 / #31110).
"""

from __future__ import annotations

import logging
from typing import Any, Dict, Optional

logger = logging.getLogger(__name__)


def _is_transient_network_error(exc: BaseException) -> bool:
"""Return True for transient network errors safe to log + swallow.

The crash class targeted by #31066 / #31110: an unhandled Telegram
``TimedOut`` (or peer ``NetworkError`` / ``httpx`` connection error)
propagating to the event loop and killing the entire gateway
process. These are by definition transient — the next poll cycle or
user action recovers — so they must never crash the process.

Walk the exception cause chain so wrapped errors (e.g. PTB's
``NetworkError`` wrapping ``httpx.ConnectError``) are still
classified. The chain is bounded to avoid pathological cycles.
"""
seen: set[int] = set()
cur: Optional[BaseException] = exc
depth = 0
transient_class_names = {
"TimedOut",
"NetworkError",
"ReadError",
"WriteError",
"ConnectError",
"ConnectTimeout",
"ReadTimeout",
"WriteTimeout",
"PoolTimeout",
"RemoteProtocolError",
"ServerDisconnectedError",
"ClientConnectorError",
"ClientOSError",
}
while cur is not None and depth < 12:
ident = id(cur)
if ident in seen:
break
seen.add(ident)
depth += 1
name = type(cur).__name__
if name in transient_class_names:
return True
cur = cur.__cause__ or cur.__context__
return False


def _gateway_loop_exception_handler(
loop: "asyncio.AbstractEventLoop", context: Dict[str, Any]
) -> None:
"""Loop-level safety net for transient network errors.

Installed once during :func:`start_gateway`. Catches the
``telegram.error.TimedOut`` crash class (issues #31066 / #31110)
and any peer transient network error before it can kill the
gateway process. Logs at WARNING with full traceback so the
originating call site stays diagnosable; non-transient errors
are forwarded to the default loop handler so real bugs still
surface.
"""
exc = context.get("exception")
if exc is not None and _is_transient_network_error(exc):
task = context.get("future") or context.get("task")
task_name = ""
if task is not None:
try:
task_name = task.get_name() if hasattr(task, "get_name") else repr(task)
except Exception:
task_name = repr(task)
logger.warning(
"Gateway swallowed transient network error from %s: %s: %s",
task_name or "<unknown task>",
type(exc).__name__,
exc,
exc_info=(type(exc), exc, exc.__traceback__),
)
return
# Fall back to the default handler for anything we don't recognise.
loop.default_exception_handler(context)
81 changes: 5 additions & 76 deletions gateway/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -367,82 +367,11 @@ def _seed_hygiene_system_prompt(
return bool(stored_prompt)


def _is_transient_network_error(exc: BaseException) -> bool:
"""Return True for transient network errors safe to log + swallow.

The crash class targeted by #31066 / #31110: an unhandled Telegram
``TimedOut`` (or peer ``NetworkError`` / ``httpx`` connection error)
propagating to the event loop and killing the entire gateway
process. These are by definition transient — the next poll cycle or
user action recovers — so they must never crash the process.

Walk the exception cause chain so wrapped errors (e.g. PTB's
``NetworkError`` wrapping ``httpx.ConnectError``) are still
classified. The chain is bounded to avoid pathological cycles.
"""
seen: set[int] = set()
cur: Optional[BaseException] = exc
depth = 0
transient_class_names = {
"TimedOut",
"NetworkError",
"ReadError",
"WriteError",
"ConnectError",
"ConnectTimeout",
"ReadTimeout",
"WriteTimeout",
"PoolTimeout",
"RemoteProtocolError",
"ServerDisconnectedError",
"ClientConnectorError",
"ClientOSError",
}
while cur is not None and depth < 12:
ident = id(cur)
if ident in seen:
break
seen.add(ident)
depth += 1
name = type(cur).__name__
if name in transient_class_names:
return True
cur = cur.__cause__ or cur.__context__
return False


def _gateway_loop_exception_handler(
loop: "asyncio.AbstractEventLoop", context: Dict[str, Any]
) -> None:
"""Loop-level safety net for transient network errors.

Installed once during :func:`start_gateway`. Catches the
``telegram.error.TimedOut`` crash class (issues #31066 / #31110)
and any peer transient network error before it can kill the
gateway process. Logs at WARNING with full traceback so the
originating call site stays diagnosable; non-transient errors
are forwarded to the default loop handler so real bugs still
surface.
"""
exc = context.get("exception")
if exc is not None and _is_transient_network_error(exc):
task = context.get("future") or context.get("task")
task_name = ""
if task is not None:
try:
task_name = task.get_name() if hasattr(task, "get_name") else repr(task)
except Exception:
task_name = repr(task)
logger.warning(
"Gateway swallowed transient network error from %s: %s: %s",
task_name or "<unknown task>",
type(exc).__name__,
exc,
exc_info=(type(exc), exc, exc.__traceback__),
)
return
# Fall back to the default handler for anything we don't recognise.
loop.default_exception_handler(context)
# --- Transient-error helpers (extracted to gateway.error_helpers) ---
from gateway.error_helpers import ( # noqa: E402
_gateway_loop_exception_handler,
_is_transient_network_error,
)


def _redact_gateway_user_facing_secrets(text: str) -> str:
Expand Down
Loading