From 21405ac47dd731e7bf959277f3f8c1b3da995d10 Mon Sep 17 00:00:00 2001 From: "Andrex Ibiza, MBA" <84248988+andrexibiza@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:39:19 -0500 Subject: [PATCH] refactor(gateway): extract document-cache + send-error mixins from base.py (shard s2) --- contributors/emails/andrexibiza@gmail.com | 1 + .../andrexibiza@users.noreply.github.com | 1 + gateway/platforms/base.py | 324 ++---------------- gateway/platforms/document_cache.py | 65 ++++ gateway/platforms/media_tag_parsing.py | 169 +++++++++ gateway/platforms/send_errors.py | 122 +++++++ .../test_platform_base_extracted_w1b.py | 198 +++++++++++ tests/gateway/test_s2_w1a_extraction.py | 197 +++++++++++ 8 files changed, 780 insertions(+), 297 deletions(-) create mode 100644 contributors/emails/andrexibiza@gmail.com create mode 100644 contributors/emails/andrexibiza@users.noreply.github.com create mode 100644 gateway/platforms/document_cache.py create mode 100644 gateway/platforms/media_tag_parsing.py create mode 100644 gateway/platforms/send_errors.py create mode 100644 tests/gateway/test_platform_base_extracted_w1b.py create mode 100644 tests/gateway/test_s2_w1a_extraction.py diff --git a/contributors/emails/andrexibiza@gmail.com b/contributors/emails/andrexibiza@gmail.com new file mode 100644 index 000000000000..efa930813a29 --- /dev/null +++ b/contributors/emails/andrexibiza@gmail.com @@ -0,0 +1 @@ +andrexibiza diff --git a/contributors/emails/andrexibiza@users.noreply.github.com b/contributors/emails/andrexibiza@users.noreply.github.com new file mode 100644 index 000000000000..efa930813a29 --- /dev/null +++ b/contributors/emails/andrexibiza@users.noreply.github.com @@ -0,0 +1 @@ +andrexibiza diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index c42b9160737d..60711671e3f8 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -1745,179 +1745,6 @@ def _log_safe_path(path: str) -> str: ) -def _match_extensionless_path(scan_text: str, match: "re.Match") -> Optional[Tuple[str, int]]: - """Resolve an extensionless MEDIA tag match to a validated on-disk path. - - Tries the regex-captured path first. When that fails validation, the - candidate is progressively extended forward across single spaces - (validation-gated, bounded at 8 tokens, never past a newline or a - subsequent ``MEDIA:`` keyword) so unknown-extension paths containing - spaces deliver (#24032). Returns ``(safe_path, end_offset)`` where - ``end_offset`` is the index in ``scan_text`` just past the matched path, - or ``None`` when nothing validates. - """ - raw = match.group("path") - path = _normalize_media_tag_path(raw) - if not path: - return None - safe = validate_media_delivery_path(path) - if safe: - return safe, match.end("path") - start = match.start("path") - nl = scan_text.find("\n", start) - limit = nl if nl != -1 else len(scan_text) - segment = scan_text[start:limit] - nxt = segment.find("MEDIA:", 1) - if nxt != -1: - segment = segment[:nxt] - pos = match.end("path") - start - for _ in range(8): - while pos < len(segment) and segment[pos] in " \t": - pos += 1 - if pos >= len(segment): - break - tok_end = pos - while tok_end < len(segment) and segment[tok_end] not in " \t": - tok_end += 1 - candidate = _normalize_media_tag_path(segment[:tok_end]) - safe = validate_media_delivery_path(candidate) - if safe: - return safe, start + tok_end - pos = tok_end - return None - - -def _merge_spans(spans: list) -> list: - """Merge overlapping/nested (start, end) spans so multi-pattern matches - over the same tag never double-delete adjacent text.""" - merged: list = [] - for s, e in sorted(spans): - if merged and s <= merged[-1][1]: - merged[-1] = (merged[-1][0], max(merged[-1][1], e)) - else: - merged.append((s, e)) - return merged - - -def _normalize_media_tag_path(raw: str) -> str: - path = str(raw or "").strip() - if len(path) >= 2 and path[0] == path[-1] and path[0] in "`\"'": - path = path[1:-1].strip() - return path.lstrip("`\"'").rstrip("`\"',.;:)}]") - - -def _path_lacks_deliverable_extension(path: str) -> bool: - """True when MEDIA_TAG_CLEANUP_RE's extension alternation does not cover - ``path`` — either the basename has no extension at all (Caddyfile, - Makefile, …) or the extension is not in MEDIA_DELIVERY_EXTS (.py, .log, - .weirdext, …). Such paths route through the validated delivery pass - (``validate_media_delivery_path``) instead of the unconditional one, so - every file type is deliverable (#36060) while nonexistent / denylisted - paths stay visible in the text. - """ - suffix = Path(path).suffix.lower() - return not suffix or suffix not in MEDIA_DELIVERY_EXTS - - -def _resolve_extensionless_candidate(path: str) -> Optional[str]: - """Validate a bare extensionless-branch path (no forward extension). - - Thin wrapper kept for call sites that only have the normalized path - (no scan-text context for spaced-path recovery). - """ - if not path: - return None - return validate_media_delivery_path(path) - - -def _strip_media_tag_directives(text: str) -> str: - """Remove MEDIA: tags and [[audio_as_voice]] / [[as_document]] markers. - - Protected spans (fenced code blocks, inline code holding non-deliverable - example tags, blockquotes, JSON string values) are used as a mask-locator - only — tags inside them are neither stripped nor mangled, matching - ``extract_media``'s treatment so display text and delivery agree (#16434). - """ - if ( - "MEDIA:" not in text - and "[[audio_as_voice]]" not in text - and "[[as_document]]" not in text - ): - return text - cleaned = text.replace("[[audio_as_voice]]", "").replace("[[as_document]]", "") - - # Locate real tag spans on a masked copy (offset-preserving), then delete - # exactly those spans from the unmasked text — same pattern as - # extract_media. Import-cycle-free: BasePlatformAdapter is defined later - # in this module, so resolve it lazily at call time. - masked = BasePlatformAdapter._mask_protected_spans(cleaned) - masked = BasePlatformAdapter._mask_json_string_media(masked) - - spans: list = [m.span() for m in MEDIA_TAG_CLEANUP_RE.finditer(masked)] - for match in MEDIA_EXTENSIONLESS_TAG_RE.finditer(masked): - path = _normalize_media_tag_path(match.group("path")) - if not path or not _path_lacks_deliverable_extension(path): - continue - resolved = _match_extensionless_path(masked, match) - if resolved is not None: - spans.append((match.start(), resolved[1])) - - if spans: - chars = list(cleaned) - for start, end in reversed(_merge_spans(spans)): - del chars[start:end] - cleaned = "".join(chars) - return cleaned - - -def get_document_cache_dir() -> Path: - """Return the document cache directory, creating it if it doesn't exist.""" - d = _resolve_cache_dir("DOCUMENT_CACHE_DIR", "cache/documents", "document_cache") - d.mkdir(parents=True, exist_ok=True) - return d - - -def cache_document_from_bytes(data: bytes, filename: str) -> str: - """ - Save raw document bytes to the cache and return the absolute file path. - - The cached filename preserves the original human-readable name with a - unique prefix: ``doc_{uuid12}_{original_filename}``. - - Args: - data: Raw document bytes. - filename: Original filename (e.g. "report.pdf"). - - Returns: - Absolute path to the cached document file as a string. - - Raises: - ValueError: If the sanitized path escapes the cache directory. - """ - cache_dir = get_document_cache_dir() - # Sanitize: strip directory components, null bytes, and control characters - safe_name = Path(filename).name if filename else "document" - safe_name = safe_name.replace("\x00", "").strip() - if not safe_name or safe_name in {".", ".."}: - safe_name = "document" - cached_name = f"doc_{uuid.uuid4().hex[:12]}_{safe_name}" - filepath = cache_dir / cached_name - # Final safety check: ensure path stays inside cache dir - if not filepath.resolve().is_relative_to(cache_dir.resolve()): - raise ValueError(f"Path traversal rejected: {filename!r}") - filepath.write_bytes(data) - return str(filepath) - - -def cleanup_document_cache(max_age_hours: int = 24) -> int: - """ - Delete cached documents older than *max_age_hours*. - - Returns the number of files removed. - """ - return _cleanup_cache_dir(get_document_cache_dir(), max_age_hours) - - # --------------------------------------------------------------------------- # Unified media caching # @@ -2264,114 +2091,6 @@ class SendResult: } ) -# ``not_found`` substrings split by blast radius. A *chat-level* not_found means -# the chat/user/group itself is gone, so the whole target is dead. A -# *thread/topic/message-level* not_found (a deleted forum topic, an edited-away -# message) leaves the parent chat reachable — it must NOT mark the whole chat -# dead. ``classify_send_error`` collapses both into ``"not_found"``; -# ``is_chat_level_not_found`` recovers the distinction for the dead-target path. -# See gateway.dead_targets. -_CHAT_LEVEL_NOT_FOUND_SUBSTRINGS = ("chat not found",) -_SUBCHAT_NOT_FOUND_SUBSTRINGS = ( - "message to edit not found", - "message to reply not found", - "thread not found", - "topic_deleted", - "message_id_invalid", -) - - -def _error_blob(exc: Optional[BaseException] = None, error_text: str = "") -> str: - """Build the lowercased text blob both send-error classifiers match against. - - Single source of truth so ``classify_send_error`` and - ``is_chat_level_not_found`` can never drift (e.g. one including the - exception class name and the other not) and silently disagree on the same - failure. Includes ``str(exc)`` (when non-empty) and the exception's class - name, plus any explicit ``error_text``. - """ - parts = [] - if error_text: - parts.append(error_text) - if exc is not None: - exc_str = str(exc) - if exc_str: - parts.append(exc_str) - parts.append(exc.__class__.__name__) - return " ".join(parts).lower() - - -def classify_send_error(exc: Optional[BaseException], error_text: str = "") -> str: - """Map a send exception / error string to a :data:`SEND_ERROR_KINDS` value. - - Platform-neutral: matches on the lowercased text of ``exc`` (and/or the - explicit ``error_text``) against the substrings the major messaging APIs - use. Conservative — anything unrecognized returns ``"unknown"`` so callers - never mistake an unclassified failure for a benign one. - """ - blob = _error_blob(exc, error_text) - if not blob.strip(): - return "unknown" - if "message_too_long" in blob or "too long" in blob or "message is too long" in blob: - return "too_long" - if ( - "can't parse entities" in blob - or "cant parse entities" in blob - or "can't find end" in blob - or "unsupported start tag" in blob - or ("entity" in blob and "parse" in blob) - or ("bad request" in blob and "entit" in blob) - ): - return "bad_format" - if ( - "forbidden" in blob - or "bot was blocked" in blob - or "blocked by the user" in blob - or "user is deactivated" in blob - or "not enough rights" in blob - or "have no rights" in blob - or "not a member" in blob - ): - return "forbidden" - if any(s in blob for s in _CHAT_LEVEL_NOT_FOUND_SUBSTRINGS) or any( - s in blob for s in _SUBCHAT_NOT_FOUND_SUBSTRINGS - ): - return "not_found" - if ( - "flood" in blob - or "too many requests" in blob - or "retry after" in blob - or "rate limit" in blob - ): - return "rate_limited" - for pat in _RETRYABLE_ERROR_PATTERNS: - if pat in blob: - return "transient" - if "connecttimeout" in blob: - return "transient" - return "unknown" - - -def is_chat_level_not_found(exc: Optional[BaseException] = None, error_text: str = "") -> bool: - """Whether a ``not_found`` failure means the *whole chat* is gone. - - :func:`classify_send_error` collapses chat-level and thread/topic/message-level - not_found into the single ``"not_found"`` kind. Only the chat-level case (the - chat/user/group no longer exists) should mark a delivery target dead; a deleted - forum topic or an edited-away message leaves the parent chat reachable. When - both a chat-level and a sub-chat marker are present, the sub-chat reading wins - (conservative: never kill a chat that may still be reachable). - - Argument order mirrors :func:`classify_send_error` (``exc`` first) and both - share :func:`_error_blob`, so the two classifiers cannot disagree on the same - failure. - """ - blob = _error_blob(exc, error_text) - if any(s in blob for s in _SUBCHAT_NOT_FOUND_SUBSTRINGS): - return False - return any(s in blob for s in _CHAT_LEVEL_NOT_FOUND_SUBSTRINGS) - - class EphemeralReply(str): """System-notice reply that auto-deletes after a TTL. @@ -2610,22 +2329,6 @@ def resolve_channel_skills( return None -def _strip_media_directives(text: str) -> str: - """Strip internal delivery directives ([[audio_as_voice]], [[as_document]], - MEDIA:) so they never render as visible text. - - Backstop only: run ``extract_media`` first. MEDIA cleanup uses the shared - ``MEDIA_TAG_CLEANUP_RE`` (only tags whose path has a known deliverable - extension are removed; an unknown-extension tag is intentionally left so the - bare-path detector downstream can still pick it up, per #34517). Validated - extension-less tags (e.g. ``MEDIA:/output/Caddyfile``) are also removed. - [[...]] is exact. - """ - if not text: - return text - return _strip_media_tag_directives(text) - - class BasePlatformAdapter(ABC): """ Base class for platform adapters. @@ -6859,3 +6562,30 @@ def truncate_message( ] return chunks + + +# --------------------------------------------------------------------------- +# Wave-1 shard-s2 extractions (verbatim moves) — re-export the moved helpers +# so existing ``from gateway.platforms.base import ...`` call sites (adapters, +# gateway/run.py, tests) keep working unchanged. Imported at the bottom of +# the module because the extracted modules import shared helpers back from +# here (cycle break; see media_tag_parsing.py / document_cache.py docstrings). +# --------------------------------------------------------------------------- +from gateway.platforms.document_cache import ( + cache_document_from_bytes, + cleanup_document_cache, + get_document_cache_dir, +) +from gateway.platforms.media_tag_parsing import ( + _match_extensionless_path, + _merge_spans, + _normalize_media_tag_path, + _path_lacks_deliverable_extension, + _resolve_extensionless_candidate, + _strip_media_directives, + _strip_media_tag_directives, +) +from gateway.platforms.send_errors import ( + classify_send_error, + is_chat_level_not_found, +) diff --git a/gateway/platforms/document_cache.py b/gateway/platforms/document_cache.py new file mode 100644 index 000000000000..9f9941185b01 --- /dev/null +++ b/gateway/platforms/document_cache.py @@ -0,0 +1,65 @@ +"""Document cache helpers for gateway platform adapters. + +Extracted from ``gateway/platforms/base.py`` (god-file decomposition +campaign, wave 1 — shard s2, cluster c3, 12 move votes). Functions moved +verbatim; ``base.py`` re-exports them. ``_resolve_cache_dir`` and +``_cleanup_cache_dir`` stay in ``base.py`` (the image/audio/video cache +helpers that remain there still use them) and are imported here at the +bottom of this module (cycle break). +""" + +import uuid +from pathlib import Path + +def get_document_cache_dir() -> Path: + """Return the document cache directory, creating it if it doesn't exist.""" + d = _resolve_cache_dir("DOCUMENT_CACHE_DIR", "cache/documents", "document_cache") + d.mkdir(parents=True, exist_ok=True) + return d + + +def cache_document_from_bytes(data: bytes, filename: str) -> str: + """ + Save raw document bytes to the cache and return the absolute file path. + + The cached filename preserves the original human-readable name with a + unique prefix: ``doc_{uuid12}_{original_filename}``. + + Args: + data: Raw document bytes. + filename: Original filename (e.g. "report.pdf"). + + Returns: + Absolute path to the cached document file as a string. + + Raises: + ValueError: If the sanitized path escapes the cache directory. + """ + cache_dir = get_document_cache_dir() + # Sanitize: strip directory components, null bytes, and control characters + safe_name = Path(filename).name if filename else "document" + safe_name = safe_name.replace("\x00", "").strip() + if not safe_name or safe_name in {".", ".."}: + safe_name = "document" + cached_name = f"doc_{uuid.uuid4().hex[:12]}_{safe_name}" + filepath = cache_dir / cached_name + # Final safety check: ensure path stays inside cache dir + if not filepath.resolve().is_relative_to(cache_dir.resolve()): + raise ValueError(f"Path traversal rejected: {filename!r}") + filepath.write_bytes(data) + return str(filepath) + + +def cleanup_document_cache(max_age_hours: int = 24) -> int: + """ + Delete cached documents older than *max_age_hours*. + + Returns the number of files removed. + """ + return _cleanup_cache_dir(get_document_cache_dir(), max_age_hours) + + +from gateway.platforms.base import ( # noqa: E402 + _cleanup_cache_dir, + _resolve_cache_dir, +) diff --git a/gateway/platforms/media_tag_parsing.py b/gateway/platforms/media_tag_parsing.py new file mode 100644 index 000000000000..c16327718b2b --- /dev/null +++ b/gateway/platforms/media_tag_parsing.py @@ -0,0 +1,169 @@ +"""MEDIA: tag parsing and directive stripping helpers. + +Extracted from ``gateway/platforms/base.py`` (god-file decomposition +campaign, wave 1 — shard s2, cluster c2, 21 move votes). Every function is +moved verbatim; ``base.py`` re-exports them so ``from +gateway.platforms.base import ...`` call sites are unchanged. The shared +regex constants (``MEDIA_TAG_CLEANUP_RE``, ``MEDIA_EXTENSIONLESS_TAG_RE``, +``MEDIA_DELIVERY_EXTS``), ``validate_media_delivery_path`` and the +``BasePlatformAdapter._mask_*`` class helpers stay in ``base.py`` (class +methods and tests still reference them there) and are imported here at the +bottom of this module — the same cycle-avoidance pattern documented in +``gateway/authz_mixin.py``. +""" + +import re +from pathlib import Path +from typing import Optional, Tuple + +def _match_extensionless_path(scan_text: str, match: "re.Match") -> Optional[Tuple[str, int]]: + """Resolve an extensionless MEDIA tag match to a validated on-disk path. + + Tries the regex-captured path first. When that fails validation, the + candidate is progressively extended forward across single spaces + (validation-gated, bounded at 8 tokens, never past a newline or a + subsequent ``MEDIA:`` keyword) so unknown-extension paths containing + spaces deliver (#24032). Returns ``(safe_path, end_offset)`` where + ``end_offset`` is the index in ``scan_text`` just past the matched path, + or ``None`` when nothing validates. + """ + raw = match.group("path") + path = _normalize_media_tag_path(raw) + if not path: + return None + safe = validate_media_delivery_path(path) + if safe: + return safe, match.end("path") + start = match.start("path") + nl = scan_text.find("\n", start) + limit = nl if nl != -1 else len(scan_text) + segment = scan_text[start:limit] + nxt = segment.find("MEDIA:", 1) + if nxt != -1: + segment = segment[:nxt] + pos = match.end("path") - start + for _ in range(8): + while pos < len(segment) and segment[pos] in " \t": + pos += 1 + if pos >= len(segment): + break + tok_end = pos + while tok_end < len(segment) and segment[tok_end] not in " \t": + tok_end += 1 + candidate = _normalize_media_tag_path(segment[:tok_end]) + safe = validate_media_delivery_path(candidate) + if safe: + return safe, start + tok_end + pos = tok_end + return None + + +def _merge_spans(spans: list) -> list: + """Merge overlapping/nested (start, end) spans so multi-pattern matches + over the same tag never double-delete adjacent text.""" + merged: list = [] + for s, e in sorted(spans): + if merged and s <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], e)) + else: + merged.append((s, e)) + return merged + + +def _normalize_media_tag_path(raw: str) -> str: + path = str(raw or "").strip() + if len(path) >= 2 and path[0] == path[-1] and path[0] in "`\"'": + path = path[1:-1].strip() + return path.lstrip("`\"'").rstrip("`\"',.;:)}]") + + +def _path_lacks_deliverable_extension(path: str) -> bool: + """True when MEDIA_TAG_CLEANUP_RE's extension alternation does not cover + ``path`` — either the basename has no extension at all (Caddyfile, + Makefile, …) or the extension is not in MEDIA_DELIVERY_EXTS (.py, .log, + .weirdext, …). Such paths route through the validated delivery pass + (``validate_media_delivery_path``) instead of the unconditional one, so + every file type is deliverable (#36060) while nonexistent / denylisted + paths stay visible in the text. + """ + suffix = Path(path).suffix.lower() + return not suffix or suffix not in MEDIA_DELIVERY_EXTS + + +def _resolve_extensionless_candidate(path: str) -> Optional[str]: + """Validate a bare extensionless-branch path (no forward extension). + + Thin wrapper kept for call sites that only have the normalized path + (no scan-text context for spaced-path recovery). + """ + if not path: + return None + return validate_media_delivery_path(path) + + +def _strip_media_tag_directives(text: str) -> str: + """Remove MEDIA: tags and [[audio_as_voice]] / [[as_document]] markers. + + Protected spans (fenced code blocks, inline code holding non-deliverable + example tags, blockquotes, JSON string values) are used as a mask-locator + only — tags inside them are neither stripped nor mangled, matching + ``extract_media``'s treatment so display text and delivery agree (#16434). + """ + if ( + "MEDIA:" not in text + and "[[audio_as_voice]]" not in text + and "[[as_document]]" not in text + ): + return text + cleaned = text.replace("[[audio_as_voice]]", "").replace("[[as_document]]", "") + + # Locate real tag spans on a masked copy (offset-preserving), then delete + # exactly those spans from the unmasked text — same pattern as + # extract_media. Import-cycle-free: BasePlatformAdapter is defined later + # in this module, so resolve it lazily at call time. + masked = BasePlatformAdapter._mask_protected_spans(cleaned) + masked = BasePlatformAdapter._mask_json_string_media(masked) + + spans: list = [m.span() for m in MEDIA_TAG_CLEANUP_RE.finditer(masked)] + for match in MEDIA_EXTENSIONLESS_TAG_RE.finditer(masked): + path = _normalize_media_tag_path(match.group("path")) + if not path or not _path_lacks_deliverable_extension(path): + continue + resolved = _match_extensionless_path(masked, match) + if resolved is not None: + spans.append((match.start(), resolved[1])) + + if spans: + chars = list(cleaned) + for start, end in reversed(_merge_spans(spans)): + del chars[start:end] + cleaned = "".join(chars) + return cleaned + + +def _strip_media_directives(text: str) -> str: + """Strip internal delivery directives ([[audio_as_voice]], [[as_document]], + MEDIA:) so they never render as visible text. + + Backstop only: run ``extract_media`` first. MEDIA cleanup uses the shared + ``MEDIA_TAG_CLEANUP_RE`` (only tags whose path has a known deliverable + extension are removed; an unknown-extension tag is intentionally left so the + bare-path detector downstream can still pick it up, per #34517). Validated + extension-less tags (e.g. ``MEDIA:/output/Caddyfile``) are also removed. + [[...]] is exact. + """ + if not text: + return text + return _strip_media_tag_directives(text) + + +# Imported at the bottom so this module never triggers ``base.py``'s own +# import of this module mid-execution (cycle break). All names below are +# referenced only at call time by the moved functions. +from gateway.platforms.base import ( # noqa: E402 + MEDIA_DELIVERY_EXTS, + MEDIA_EXTENSIONLESS_TAG_RE, + MEDIA_TAG_CLEANUP_RE, + BasePlatformAdapter, + validate_media_delivery_path, +) diff --git a/gateway/platforms/send_errors.py b/gateway/platforms/send_errors.py new file mode 100644 index 000000000000..1558aeb9ebd3 --- /dev/null +++ b/gateway/platforms/send_errors.py @@ -0,0 +1,122 @@ +"""Send-error classification helpers for gateway platforms. + +Extracted verbatim from ``gateway/platforms/base.py`` (shard s2, cluster c8: +_error_blob, classify_send_error, is_chat_level_not_found, plus the +chat/sub-chat not-found substring constants). +``gateway/platforms/base.py`` re-exports the public names so existing +import sites keep working unchanged. +""" + +from typing import Optional + + +# ``not_found`` substrings split by blast radius. A *chat-level* not_found means +# the chat/user/group itself is gone, so the whole target is dead. A +# *thread/topic/message-level* not_found (a deleted forum topic, an edited-away +# message) leaves the parent chat reachable — it must NOT mark the whole chat +# dead. ``classify_send_error`` collapses both into ``"not_found"``; +# ``is_chat_level_not_found`` recovers the distinction for the dead-target path. +# See gateway.dead_targets. +_CHAT_LEVEL_NOT_FOUND_SUBSTRINGS = ("chat not found",) +_SUBCHAT_NOT_FOUND_SUBSTRINGS = ( + "message to edit not found", + "message to reply not found", + "thread not found", + "topic_deleted", + "message_id_invalid", +) + + +def _error_blob(exc: Optional[BaseException] = None, error_text: str = "") -> str: + """Build the lowercased text blob both send-error classifiers match against. + + Single source of truth so ``classify_send_error`` and + ``is_chat_level_not_found`` can never drift (e.g. one including the + exception class name and the other not) and silently disagree on the same + failure. Includes ``str(exc)`` (when non-empty) and the exception's class + name, plus any explicit ``error_text``. + """ + parts = [] + if error_text: + parts.append(error_text) + if exc is not None: + exc_str = str(exc) + if exc_str: + parts.append(exc_str) + parts.append(exc.__class__.__name__) + return " ".join(parts).lower() + + +def classify_send_error(exc: Optional[BaseException], error_text: str = "") -> str: + """Map a send exception / error string to a :data:`SEND_ERROR_KINDS` value. + + Platform-neutral: matches on the lowercased text of ``exc`` (and/or the + explicit ``error_text``) against the substrings the major messaging APIs + use. Conservative — anything unrecognized returns ``"unknown"`` so callers + never mistake an unclassified failure for a benign one. + """ + # Local import: _RETRYABLE_ERROR_PATTERNS stays in base.py (also used by + # BasePlatformAdapter._is_retryable_error); module-level import would + # cycle with base.py's re-export of this module. + from gateway.platforms.base import _RETRYABLE_ERROR_PATTERNS + blob = _error_blob(exc, error_text) + if not blob.strip(): + return "unknown" + if "message_too_long" in blob or "too long" in blob or "message is too long" in blob: + return "too_long" + if ( + "can't parse entities" in blob + or "cant parse entities" in blob + or "can't find end" in blob + or "unsupported start tag" in blob + or ("entity" in blob and "parse" in blob) + or ("bad request" in blob and "entit" in blob) + ): + return "bad_format" + if ( + "forbidden" in blob + or "bot was blocked" in blob + or "blocked by the user" in blob + or "user is deactivated" in blob + or "not enough rights" in blob + or "have no rights" in blob + or "not a member" in blob + ): + return "forbidden" + if any(s in blob for s in _CHAT_LEVEL_NOT_FOUND_SUBSTRINGS) or any( + s in blob for s in _SUBCHAT_NOT_FOUND_SUBSTRINGS + ): + return "not_found" + if ( + "flood" in blob + or "too many requests" in blob + or "retry after" in blob + or "rate limit" in blob + ): + return "rate_limited" + for pat in _RETRYABLE_ERROR_PATTERNS: + if pat in blob: + return "transient" + if "connecttimeout" in blob: + return "transient" + return "unknown" + + +def is_chat_level_not_found(exc: Optional[BaseException] = None, error_text: str = "") -> bool: + """Whether a ``not_found`` failure means the *whole chat* is gone. + + :func:`classify_send_error` collapses chat-level and thread/topic/message-level + not_found into the single ``"not_found"`` kind. Only the chat-level case (the + chat/user/group no longer exists) should mark a delivery target dead; a deleted + forum topic or an edited-away message leaves the parent chat reachable. When + both a chat-level and a sub-chat marker are present, the sub-chat reading wins + (conservative: never kill a chat that may still be reachable). + + Argument order mirrors :func:`classify_send_error` (``exc`` first) and both + share :func:`_error_blob`, so the two classifiers cannot disagree on the same + failure. + """ + blob = _error_blob(exc, error_text) + if any(s in blob for s in _SUBCHAT_NOT_FOUND_SUBSTRINGS): + return False + return any(s in blob for s in _CHAT_LEVEL_NOT_FOUND_SUBSTRINGS) diff --git a/tests/gateway/test_platform_base_extracted_w1b.py b/tests/gateway/test_platform_base_extracted_w1b.py new file mode 100644 index 000000000000..ed714744f4b4 --- /dev/null +++ b/tests/gateway/test_platform_base_extracted_w1b.py @@ -0,0 +1,198 @@ +"""Regression tests for the wave-1 extraction of ``gateway/platforms/base.py``. + +Shard s2 move clusters covered (verbatim extraction): + +* ``c2`` MEDIA: tag parsing / directive stripping + -> ``gateway/platforms/media_tag_parsing.py`` (21 move votes) +* ``c3`` document cache helpers + -> ``gateway/platforms/document_cache.py`` (12 move votes) + +Two contracts are asserted here: + +1. Behavior of the moved pure helpers (unchanged semantics). +2. Re-export parity: every moved function is still importable from + ``gateway.platforms.base`` and is the *same object* as the one in its new + module, so all existing ``from gateway.platforms.base import ...`` call + sites (adapters, media_cache, run.py, tests) keep working. +""" + +import os +import re +import time +from pathlib import Path + +import pytest + +from gateway.platforms.base import ( + MEDIA_EXTENSIONLESS_TAG_RE, + MEDIA_TAG_CLEANUP_RE, + _match_extensionless_path, + _merge_spans, + _normalize_media_tag_path, + _path_lacks_deliverable_extension, + _resolve_extensionless_candidate, + _strip_media_directives, + _strip_media_tag_directives, + cache_document_from_bytes, + cleanup_document_cache, + get_document_cache_dir, +) +from gateway.platforms.document_cache import ( + cache_document_from_bytes as dc_cache_document_from_bytes, + cleanup_document_cache as dc_cleanup_document_cache, + get_document_cache_dir as dc_get_document_cache_dir, +) +from gateway.platforms.media_tag_parsing import ( + _match_extensionless_path as mtp_match_extensionless_path, + _merge_spans as mtp_merge_spans, + _normalize_media_tag_path as mtp_normalize_media_tag_path, + _path_lacks_deliverable_extension as mtp_path_lacks_deliverable_extension, + _resolve_extensionless_candidate as mtp_resolve_extensionless_candidate, + _strip_media_directives as mtp_strip_media_directives, + _strip_media_tag_directives as mtp_strip_media_tag_directives, +) + +# --------------------------------------------------------------------------- +# Re-export parity: base re-exports the same objects the new modules define +# --------------------------------------------------------------------------- + +PARITY_PAIRS = [ + (_match_extensionless_path, mtp_match_extensionless_path), + (_merge_spans, mtp_merge_spans), + (_normalize_media_tag_path, mtp_normalize_media_tag_path), + (_path_lacks_deliverable_extension, mtp_path_lacks_deliverable_extension), + (_resolve_extensionless_candidate, mtp_resolve_extensionless_candidate), + (_strip_media_directives, mtp_strip_media_directives), + (_strip_media_tag_directives, mtp_strip_media_tag_directives), + (cache_document_from_bytes, dc_cache_document_from_bytes), + (cleanup_document_cache, dc_cleanup_document_cache), + (get_document_cache_dir, dc_get_document_cache_dir), +] + + +@pytest.mark.parametrize("base_fn,module_fn", PARITY_PAIRS) +def test_reexport_parity(base_fn, module_fn): + assert base_fn is module_fn + + +# --------------------------------------------------------------------------- +# MEDIA: tag parsing helpers (cluster c2) +# --------------------------------------------------------------------------- + +def test_normalize_media_tag_path_strips_quotes_and_punctuation(): + assert _normalize_media_tag_path("`/tmp/x.png`") == "/tmp/x.png" + assert _normalize_media_tag_path('"/tmp/x.png"') == "/tmp/x.png" + assert _normalize_media_tag_path("/tmp/x.png,;") == "/tmp/x.png" + assert _normalize_media_tag_path("") == "" + assert _normalize_media_tag_path(None) == "" + + +def test_merge_spans_merges_overlapping_and_nested(): + assert _merge_spans([(1, 3), (2, 5)]) == [(1, 5)] + assert _merge_spans([(1, 5), (2, 3)]) == [(1, 5)] + assert _merge_spans([(0, 2), (4, 6)]) == [(0, 2), (4, 6)] + assert _merge_spans([]) == [] + + +def test_path_lacks_deliverable_extension(): + assert _path_lacks_deliverable_extension("Caddyfile") is True + assert _path_lacks_deliverable_extension("Makefile") is True + assert _path_lacks_deliverable_extension("notes.log") is True + assert _path_lacks_deliverable_extension("photo.png") is False + assert _path_lacks_deliverable_extension("report.pdf") is False + + +def test_resolve_extensionless_candidate(): + assert _resolve_extensionless_candidate("") is None + assert _resolve_extensionless_candidate(None) is None + + +def test_strip_media_tag_directives_removes_tags_and_markers(): + text = "see [[audio_as_voice]] MEDIA:/tmp/a.png for the file" + cleaned = _strip_media_tag_directives(text) + assert "MEDIA:" not in cleaned + assert "[[audio_as_voice]]" not in cleaned + + +def test_strip_media_tag_directives_passthrough_when_nothing_to_strip(): + text = "plain text, no directives" + assert _strip_media_tag_directives(text) == text + + +def test_strip_media_directives_delegates(): + text = "x [[as_document]] MEDIA:/tmp/b.png y" + cleaned = _strip_media_directives(text) + assert "MEDIA:" not in cleaned + assert "[[as_document]]" not in cleaned + assert _strip_media_directives("") == "" + + +def test_match_extensionless_path_rejects_unknown_match(): + # A MEDIA_EXTENSIONLESS_TAG_RE match whose path fails validation (the + # path does not exist on disk) must resolve to None, never to a bogus + # path — the validation oracle is the contract (#24032). + match = MEDIA_EXTENSIONLESS_TAG_RE.match("MEDIA:/definitely/not/here.txt") + assert match is not None + assert _match_extensionless_path("MEDIA:/definitely/not/here.txt", match) is None + + +def test_regex_constants_still_exported_from_base(): + assert MEDIA_TAG_CLEANUP_RE.search("MEDIA:/x.png") is not None + assert MEDIA_EXTENSIONLESS_TAG_RE.search("MEDIA:/Caddyfile") is not None + + +# --------------------------------------------------------------------------- +# Document cache helpers (cluster c3) +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def _redirect_document_cache(tmp_path, monkeypatch): + """Point the module-level DOCUMENT_CACHE_DIR to a fresh tmp_path.""" + monkeypatch.setattr( + "gateway.platforms.base.DOCUMENT_CACHE_DIR", tmp_path / "doc_cache" + ) + + +def test_get_document_cache_dir_creates_directory(tmp_path): + d = get_document_cache_dir() + assert isinstance(d, Path) + assert d.is_dir() + assert d == (tmp_path / "doc_cache").resolve() or d == tmp_path / "doc_cache" + + +def test_cache_document_from_bytes_writes_prefixed_file(): + path = cache_document_from_bytes(b"hello", "report.pdf") + assert Path(path).is_file() + name = Path(path).name + assert name.startswith("doc_") + assert name.endswith("_report.pdf") + assert Path(path).read_bytes() == b"hello" + + +def test_cache_document_from_bytes_sanitizes_filename(): + path = cache_document_from_bytes(b"data", "") + assert Path(path).is_file() + assert Path(path).name.startswith("doc_") + assert Path(path).name.endswith("_document") + + +def test_cache_document_from_bytes_strips_directory_components(): + # The sanitizer keeps only the basename, so a traversal-looking filename + # can never escape the cache dir (the ValueError guard is a backstop). + path = cache_document_from_bytes(b"data", "../../escape.txt") + assert "/" not in Path(path).name and "\\" not in Path(path).name + assert Path(path).is_file() + + +def test_cleanup_document_cache_removes_only_old_files(): + cache_dir = get_document_cache_dir() + old = cache_dir / "doc_old.bin" + new = cache_dir / "doc_new.bin" + old.write_bytes(b"x") + new.write_bytes(b"y") + cutoff = time.time() - (25 * 3600) + os.utime(old, (cutoff, cutoff)) + removed = cleanup_document_cache(max_age_hours=24) + assert removed == 1 + assert not old.exists() + assert new.exists() diff --git a/tests/gateway/test_s2_w1a_extraction.py b/tests/gateway/test_s2_w1a_extraction.py new file mode 100644 index 000000000000..cf3299b4a89b --- /dev/null +++ b/tests/gateway/test_s2_w1a_extraction.py @@ -0,0 +1,197 @@ +"""Regression tests for the s2-w1a extraction of gateway/platforms/base.py. + +Wave-1 blind implementation moved two unanimous move-clusters out of the +god-file into sibling modules (per shard-plan s2): + +* cluster c3 -> ``gateway/platforms/document_cache.py`` + (get_document_cache_dir, cache_document_from_bytes, cleanup_document_cache) +* cluster c8 -> ``gateway/platforms/send_errors.py`` + (_error_blob, classify_send_error, is_chat_level_not_found) + +``gateway/platforms/base.py`` re-exports the public names so existing import +sites (adapters, gateway/delivery.py, tests) keep working unchanged. These +tests pin the re-export identity and the pure behavior of the moved code. +""" + +import os +import time +from pathlib import Path + +import pytest + +from gateway.platforms import base as base_mod +from gateway.platforms.base import ( + SEND_ERROR_KINDS, + cache_document_from_bytes, + classify_send_error, + cleanup_document_cache, + get_document_cache_dir, + is_chat_level_not_found, +) +from gateway.platforms.document_cache import ( + cache_document_from_bytes as moved_cache_document, + cleanup_document_cache as moved_cleanup_document, + get_document_cache_dir as moved_get_doc_dir, +) +from gateway.platforms.send_errors import ( + _error_blob, + classify_send_error as moved_classify, + is_chat_level_not_found as moved_chat_level, +) + + +# --------------------------------------------------------------------------- +# Re-export identity: base.py must keep exposing the moved names as the SAME +# objects (existing callers import them from gateway.platforms.base). +# --------------------------------------------------------------------------- + +class TestBaseReExports: + def test_document_cache_names_are_the_moved_objects(self): + assert base_mod.get_document_cache_dir is moved_get_doc_dir + assert base_mod.cache_document_from_bytes is moved_cache_document + assert base_mod.cleanup_document_cache is moved_cleanup_document + + def test_send_error_names_are_the_moved_objects(self): + assert base_mod.classify_send_error is moved_classify + assert base_mod.is_chat_level_not_found is moved_chat_level + assert base_mod.SEND_ERROR_KINDS is SEND_ERROR_KINDS + + +# --------------------------------------------------------------------------- +# Send-error classification (cluster c8) — pure behavior in the new module +# --------------------------------------------------------------------------- + +class TestClassifySendError: + @pytest.mark.parametrize( + "text,expected", + [ + ("Message_too_long", "too_long"), + ("Bad Request: message is too long", "too_long"), + ("Bad Request: can't parse entities: unsupported start tag", "bad_format"), + ("Bad Request: can't find end of the entity", "bad_format"), + ("Forbidden: bot was blocked by the user", "forbidden"), + ("Forbidden: user is deactivated", "forbidden"), + ("Bad Request: not enough rights to send text messages", "forbidden"), + ("Bad Request: chat not found", "not_found"), + ("Bad Request: message to edit not found", "not_found"), + ("Too Many Requests: retry after 12", "rate_limited"), + ("Flood control exceeded", "rate_limited"), + ("ConnectError: connection refused", "transient"), + ("ConnectTimeout", "transient"), + ("some entirely novel provider message", "unknown"), + ("", "unknown"), + ], + ) + def test_classify_send_error_text(self, text, expected): + assert classify_send_error(None, text) == expected + # The moved module must agree with the base re-export exactly. + assert moved_classify(None, text) == expected + + def test_every_classification_is_in_the_vocabulary(self): + for s in [ + "message_too_long", + "can't parse entities", + "forbidden", + "chat not found", + "flood", + "connecterror", + "mystery", + "", + ]: + assert classify_send_error(None, s) in SEND_ERROR_KINDS + + def test_retryable_patterns_still_resolve_from_base(self): + # _RETRYABLE_ERROR_PATTERNS lives in base.py; classify_send_error + # must still see it through the local import. + assert classify_send_error(None, "ConnectionResetError: broken pipe") == "transient" + assert classify_send_error(None, "RemoteDisconnected") == "transient" + + +class TestIsChatLevelNotFound: + def test_chat_level_is_true(self): + assert is_chat_level_not_found(None, "Bad Request: chat not found") is True + + def test_subchat_only_is_false(self): + assert is_chat_level_not_found(None, "Bad Request: message to edit not found") is False + assert is_chat_level_not_found(None, "thread not found") is False + + def test_subchat_wins_when_both_present(self): + # Conservative: a sub-chat marker means the parent chat may still be + # reachable, so the target must NOT be marked dead. + assert ( + is_chat_level_not_found(None, "chat not found: message to edit not found") + is False + ) + + def test_matches_agree_with_classifier(self): + # classify_send_error collapses both families into "not_found". + assert classify_send_error(None, "Bad Request: chat not found") == "not_found" + assert classify_send_error(None, "message to edit not found") == "not_found" + + +class TestErrorBlob: + def test_includes_exc_class_and_text_lowercased(self): + exc = ValueError("Bad Request: chat not found") + blob = _error_blob(exc) + assert blob == "bad request: chat not found valueerror" + + def test_error_text_only(self): + assert _error_blob(None, "Forbidden: bot was blocked") == "forbidden: bot was blocked" + + def test_empty(self): + assert _error_blob() == "" + + +# --------------------------------------------------------------------------- +# Document cache (cluster c3) — pure behavior in the new module +# --------------------------------------------------------------------------- + +@pytest.fixture(autouse=True) +def _redirect_cache(tmp_path, monkeypatch): + """Point the module-level DOCUMENT_CACHE_DIR at a fresh tmp_path (same + seam the original tests/gateway/test_document_cache.py uses).""" + monkeypatch.setattr( + "gateway.platforms.base.DOCUMENT_CACHE_DIR", tmp_path / "doc_cache" + ) + + +class TestGetDocumentCacheDir: + def test_creates_directory(self): + cache_dir = get_document_cache_dir() + assert cache_dir.exists() + assert cache_dir.is_dir() + assert get_document_cache_dir() is not None + + +class TestCacheDocumentFromBytes: + def test_basic_caching(self): + data = b"hello world" + path = cache_document_from_bytes(data, "test.txt") + assert os.path.exists(path) + assert Path(path).read_bytes() == data + + def test_filename_preserved_in_path(self): + path = cache_document_from_bytes(b"data", "report.pdf") + assert "report.pdf" in os.path.basename(path) + assert os.path.basename(path).startswith("doc_") + + def test_empty_filename_uses_fallback(self): + path = cache_document_from_bytes(b"data", "") + assert "document" in os.path.basename(path) + + def test_moved_module_agrees(self): + path = moved_cache_document(b"data", "moved.txt") + assert "moved.txt" in os.path.basename(path) + + +class TestCleanupDocumentCache: + def test_removes_old_files(self): + cache_dir = get_document_cache_dir() + old_file = cache_dir / "old.txt" + old_file.write_text("old") + old_mtime = time.time() - 48 * 3600 + os.utime(old_file, (old_mtime, old_mtime)) + + removed = cleanup_document_cache(max_age_hours=24) + assert removed == 1 + assert not old_file.exists()