diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py index 417893fea2d4b..c8fad2c9f93f6 100644 --- a/gateway/platforms/base.py +++ b/gateway/platforms/base.py @@ -3118,6 +3118,14 @@ def truncate_message( if safe_split > _cp_limit // 4: split_at = safe_split + if _len is not len: + # Re-clamp after natural-break adjustments so the final slice + # is still valid under custom unit counting (e.g. UTF-16). + safe_prefix = _prefix_within_utf16_limit(remaining[:split_at], headroom) + split_at = len(safe_prefix) + if split_at < 1: + split_at = max(1, _custom_unit_to_cp(remaining, max(1, headroom), _len)) + chunk_body = remaining[:split_at] remaining = remaining[split_at:].lstrip() diff --git a/tests/gateway/test_platform_base.py b/tests/gateway/test_platform_base.py index a6e0d51d60eeb..14a3bb2e11cca 100644 --- a/tests/gateway/test_platform_base.py +++ b/tests/gateway/test_platform_base.py @@ -633,6 +633,18 @@ def test_code_blocks_preserved_with_utf16(self): f"Chunk {i} has unbalanced fences ({fence_count})" ) + def test_natural_break_split_keeps_utf16_safe_boundaries(self): + """Natural break preference must still produce UTF-16-safe chunks.""" + msg = ("alpha 🎩 beta " * 400).strip() + chunks = BasePlatformAdapter.truncate_message(msg, 120, len_fn=utf16_len) + assert len(chunks) > 1 + for i, chunk in enumerate(chunks): + # Regression guard: chunk should always be encodable and within limits. + chunk.encode("utf-16-le") + assert utf16_len(chunk) <= 120 + 20, ( + f"Chunk {i} UTF-16 length {utf16_len(chunk)} exceeds limit" + ) + class TestProxyKwargsForAiohttp: """Verify proxy_kwargs_for_aiohttp routes all schemes through ProxyConnector."""