From a720a2b4cfcbcfcebc8457cdef41938f7b141f82 Mon Sep 17 00:00:00 2001 From: Aubrey Freeman III Date: Thu, 16 Apr 2026 13:05:22 -0500 Subject: [PATCH 1/3] fix: use UTF-16 length for Telegram stream consumer message splitting The stream consumer measured message length using Python's len() (Unicode code points), but Telegram's actual limit is in UTF-16 code units. This caused messages with supplementary characters (emoji, CJK, etc.) to exceed Telegram's 4096-character limit, resulting in truncated messages with formatting artifacts. Changes: - Add message_len_fn property to BasePlatformAdapter (defaults to len) - Override in TelegramAdapter to return utf16_len - Stream consumer uses adapter.message_len_fn for: - safe_limit calculation - overflow detection - truncate_message calls - split point calculation (via _custom_unit_to_cp) - fallback final send chunking Fixes truncated messages with black square artifacts on Telegram when the model generates responses containing multi-byte Unicode characters. --- gateway/platforms/base.py | 9 ++++++ gateway/platforms/telegram.py | 5 ++++ gateway/stream_consumer.py | 52 +++++++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index d471818a27c9..55a53952169f 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -1311,6 +1311,15 @@ def __init__(self, config: PlatformConfig, platform: Platform): # _keep_typing skips send_typing when the chat_id is in this set. self._typing_paused: set = set() + @property + def message_len_fn(self) -> Callable[[str], int]: + """Return the length function for measuring message size on this platform. + + Override in adapters whose platform counts characters differently from + Python ``len`` (e.g. Telegram counts UTF-16 code units). + """ + return len + @property def has_fatal_error(self) -> bool: return self._fatal_error_message is not None diff --git a/gateway/platforms/telegram.py b/gateway/platforms/telegram.py index ae34ee9210ca..201912de80af 100644 --- a/gateway/platforms/telegram.py +++ b/gateway/platforms/telegram.py @@ -283,6 +283,11 @@ class TelegramAdapter(BasePlatformAdapter): MEDIA_GROUP_WAIT_SECONDS = 0.8 _GENERAL_TOPIC_THREAD_ID = "1" + @property + def message_len_fn(self): + """Telegram measures message length in UTF-16 code units.""" + return utf16_len + def __init__(self, config: PlatformConfig): super().__init__(config, Platform.TELEGRAM) self._app: Optional[Application] = None diff --git a/gateway/stream_consumer.py b/gateway/stream_consumer.py index 0539b825b837..4ef557ef997c 100644 --- a/gateway/stream_consumer.py +++ b/gateway/stream_consumer.py @@ -21,7 +21,10 @@ import re import time from dataclasses import dataclass -from typing import Any, Optional +from typing import Any, Callable, Optional + +from gateway.platforms.base import BasePlatformAdapter as _BasePlatformAdapter +from gateway.platforms.base import _custom_unit_to_cp logger = logging.getLogger("gateway.stream_consumer") @@ -301,9 +304,18 @@ def _flush_think_buffer(self) -> None: async def run(self) -> None: """Async task that drains the queue and edits the platform message.""" - # Platform message length limit — leave room for cursor + formatting + # Platform message length limit — leave room for cursor + formatting. + # Use the adapter's length function (e.g. utf16_len for Telegram) so + # overflow detection matches what the platform actually enforces. + # Gate on isinstance(BasePlatformAdapter) so test MagicMocks (whose + # auto-attributes return mock objects, not callables) fall back to len. + _len_fn: "Callable[[str], int]" = ( + self.adapter.message_len_fn + if isinstance(self.adapter, _BasePlatformAdapter) + else len + ) _raw_limit = getattr(self.adapter, "MAX_MESSAGE_LENGTH", 4096) - _safe_limit = max(500, _raw_limit - len(self.cfg.cursor) - 100) + _safe_limit = max(500, _raw_limit - _len_fn(self.cfg.cursor) - 100) try: while True: @@ -345,6 +357,10 @@ async def run(self) -> None: should_edit = should_edit or ( (elapsed >= self._current_edit_interval and self._accumulated) + # buffer_threshold is intentionally codepoint-based: + # it's a debounce heuristic ("send updates roughly + # every N visible characters"), not a platform-limit + # check. _len_fn is reserved for overflow detection. or len(self._accumulated) >= self.cfg.buffer_threshold ) @@ -353,7 +369,7 @@ async def run(self) -> None: # Split overflow: if accumulated text exceeds the platform # limit, split into properly sized chunks. if ( - len(self._accumulated) > _safe_limit + _len_fn(self._accumulated) > _safe_limit and self._message_id is None ): # No existing message to edit (first message or after a @@ -362,7 +378,7 @@ async def run(self) -> None: # proper word/code-fence boundaries and chunk # indicators like "(1/2)". chunks = self.adapter.truncate_message( - self._accumulated, _safe_limit + self._accumulated, _safe_limit, len_fn=_len_fn, ) chunks_delivered = False reply_to = self._message_id or self._initial_reply_to_id @@ -389,11 +405,14 @@ async def run(self) -> None: # Existing message: edit it with the first chunk, then # start a new message for the overflow remainder. while ( - len(self._accumulated) > _safe_limit + _len_fn(self._accumulated) > _safe_limit and self._message_id is not None and self._edit_supported ): - split_at = self._accumulated.rfind("\n", 0, _safe_limit) + _cp_budget = _custom_unit_to_cp( + self._accumulated, _safe_limit, _len_fn, + ) + split_at = self._accumulated.rfind("\n", 0, _cp_budget) if split_at < _safe_limit // 2: split_at = _safe_limit chunk = self._accumulated[:split_at] @@ -584,14 +603,18 @@ def _continuation_text(self, final_text: str) -> str: return final_text @staticmethod - def _split_text_chunks(text: str, limit: int) -> list[str]: + def _split_text_chunks( + text: str, limit: int, + len_fn: "Callable[[str], int]" = len, + ) -> list[str]: """Split text into reasonably sized chunks for fallback sends.""" - if len(text) <= limit: + if len_fn(text) <= limit: return [text] chunks: list[str] = [] remaining = text - while len(remaining) > limit: - split_at = remaining.rfind("\n", 0, limit) + while len_fn(remaining) > limit: + _cp_budget = _custom_unit_to_cp(remaining, limit, len_fn) + split_at = remaining.rfind("\n", 0, _cp_budget) if split_at < limit // 2: split_at = limit chunks.append(remaining[:split_at]) @@ -647,8 +670,13 @@ async def _send_fallback_final(self, text: str) -> None: return raw_limit = getattr(self.adapter, "MAX_MESSAGE_LENGTH", 4096) + _len_fn: "Callable[[str], int]" = ( + self.adapter.message_len_fn + if isinstance(self.adapter, _BasePlatformAdapter) + else len + ) safe_limit = max(500, raw_limit - 100) - chunks = self._split_text_chunks(continuation, safe_limit) + chunks = self._split_text_chunks(continuation, safe_limit, len_fn=_len_fn) stale_message_id = self._message_id # partial message to clean up last_message_id: Optional[str] = None From 112a30d7c9ccf183ffe2fd53c99240bd2bfdf110 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 10 May 2026 16:17:48 -0700 Subject: [PATCH 2/3] test(stream-consumer): add UTF-16 overflow regression tests for #11170 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New TestUtf16OverflowDetection class covers two scenarios: - test_emoji_text_exceeding_utf16_limit_triggers_overflow_split: feeds 2200 emoji codepoints (4400 UTF-16 units) — under Telegram's codepoint-equivalent limit but over its UTF-16 limit. Asserts truncate_message was called with len_fn=utf16_len, confirming the consumer detected the overflow. - test_codepoint_only_adapter_falls_back_to_len: documents that adapters which don't subclass BasePlatformAdapter (or test MagicMocks) fall back to plain len for backwards compat. The contributor's PR shipped no tests for the UTF-16 path. --- tests/gateway/test_stream_consumer.py | 93 +++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/tests/gateway/test_stream_consumer.py b/tests/gateway/test_stream_consumer.py index b5e423f96058..12671d806d8c 100644 --- a/tests/gateway/test_stream_consumer.py +++ b/tests/gateway/test_stream_consumer.py @@ -1638,3 +1638,96 @@ async def test_no_callback_when_none(self): await consumer.run() assert consumer.already_sent is True + + +class TestUtf16OverflowDetection: + """Regression coverage for #11170 — Telegram counts message length in + UTF-16 code units, not Python codepoints. A response with supplementary + characters (emoji, CJK in some ranges) can have len()=3000 codepoints + but utf16_len()=5000+ units, blowing past Telegram's 4096 limit.""" + + def _make_telegram_like_adapter(self): + """Construct a minimal BasePlatformAdapter subclass that overrides + message_len_fn like Telegram does.""" + from gateway.platforms.base import utf16_len, BasePlatformAdapter + + TelegramLikeAdapter = type( + "TelegramLikeAdapter", + (BasePlatformAdapter,), + { + "MAX_MESSAGE_LENGTH": 4096, + "message_len_fn": property(lambda self: utf16_len), + }, + ) + # Defeat ABCMeta abstract-instantiation guard by clearing the cached + # abstract methods set after class creation. + TelegramLikeAdapter.__abstractmethods__ = frozenset() + adapter = TelegramLikeAdapter.__new__(TelegramLikeAdapter) + adapter._typing_paused = set() + adapter._fatal_error_message = None + return adapter + + @pytest.mark.asyncio + async def test_emoji_text_exceeding_utf16_limit_triggers_overflow_split(self): + """A response that is under 4096 codepoints but over 4096 UTF-16 + units must trigger the overflow-split path.""" + from gateway.platforms.base import utf16_len + + adapter = self._make_telegram_like_adapter() + # Mock the send/edit methods we actually call + adapter.send = AsyncMock( + return_value=SimpleNamespace(success=True, message_id="msg_1"), + ) + adapter.edit_message = AsyncMock( + return_value=SimpleNamespace(success=True), + ) + # truncate_message: emit two halves so we can assert the split fired + adapter.truncate_message = MagicMock( + side_effect=lambda text, limit, **kw: [text[:len(text)//2], text[len(text)//2:]], + ) + + config = StreamConsumerConfig(edit_interval=0.01, buffer_threshold=5) + consumer = GatewayStreamConsumer(adapter, "chat_123", config) + + # 🚀 is 1 codepoint = 2 UTF-16 units. 2200 of them = 2200 codepoints, + # 4400 UTF-16 units. Under the codepoint-equivalent limit (would not + # trigger split with len()) but over Telegram's UTF-16 4096 limit. + emoji_text = "🚀" * 2200 + assert len(emoji_text) < adapter.MAX_MESSAGE_LENGTH, ( + "Test setup invariant: codepoint count under limit" + ) + assert utf16_len(emoji_text) > adapter.MAX_MESSAGE_LENGTH, ( + "Test setup invariant: UTF-16 count over limit" + ) + + consumer.on_delta(emoji_text) + task = asyncio.create_task(consumer.run()) + await asyncio.sleep(0.05) + consumer.finish() + await task + + # The fix: stream consumer detects UTF-16 overflow and calls + # truncate_message to split. Without the fix, len() would return + # 2200 (under 4096) and no split would fire — Telegram would then + # reject the send or render \x00 artifacts. + adapter.truncate_message.assert_called(), ( + "UTF-16 overflow not detected — emoji text bypassed split path" + ) + # truncate_message must have been called with len_fn=utf16_len + call_kwargs = adapter.truncate_message.call_args[1] + assert call_kwargs.get("len_fn") is utf16_len, ( + f"truncate_message called without utf16_len: {call_kwargs}" + ) + + def test_codepoint_only_adapter_falls_back_to_len(self): + """Adapters without message_len_fn override (or test MagicMocks) + must use plain len for backwards compatibility.""" + adapter = MagicMock() + adapter.MAX_MESSAGE_LENGTH = 4096 + config = StreamConsumerConfig(cursor=" ▉") + consumer = GatewayStreamConsumer(adapter, "chat_123", config) + # The isinstance guard means MagicMock adapters get len, not the + # auto-attr mock. Verified indirectly by all the other tests in + # this file passing — they all use MagicMock adapters. + assert consumer is not None + From 62065f0460a400fcbc0ea38b467d7a1ff6e6e12c Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Sun, 10 May 2026 16:17:48 -0700 Subject: [PATCH 3/3] chore: AUTHOR_MAP entry for Freeman-Consulting --- scripts/release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.py b/scripts/release.py index dd2dd1d23bc4..2d72efc22c68 100755 --- a/scripts/release.py +++ b/scripts/release.py @@ -61,6 +61,7 @@ "HuangYuChuh@users.noreply.github.com": "HuangYuChuh", "aaronwong1989@gmail.com": "hrygo", "26729613+hrygo@users.noreply.github.com": "hrygo", + "aubrey@freeman-wisco.com": "Freeman-Consulting", "ra2157218@gmail.com": "Abd0r", "abdielv@proton.me": "AJV20", "mason@growagainorchids.com": "masonjames",