Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions gateway/platforms/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -3118,6 +3118,14 @@ def truncate_message(
if safe_split > _cp_limit // 4:
split_at = safe_split

if _len is not len:
# Re-clamp after natural-break adjustments so the final slice
# is still valid under custom unit counting (e.g. UTF-16).
safe_prefix = _prefix_within_utf16_limit(remaining[:split_at], headroom)
split_at = len(safe_prefix)
if split_at < 1:
split_at = max(1, _custom_unit_to_cp(remaining, max(1, headroom), _len))

chunk_body = remaining[:split_at]
remaining = remaining[split_at:].lstrip()

Expand Down
12 changes: 12 additions & 0 deletions tests/gateway/test_platform_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,18 @@ def test_code_blocks_preserved_with_utf16(self):
f"Chunk {i} has unbalanced fences ({fence_count})"
)

def test_natural_break_split_keeps_utf16_safe_boundaries(self):
"""Natural break preference must still produce UTF-16-safe chunks."""
msg = ("alpha 🎩 beta " * 400).strip()
chunks = BasePlatformAdapter.truncate_message(msg, 120, len_fn=utf16_len)
assert len(chunks) > 1
for i, chunk in enumerate(chunks):
# Regression guard: chunk should always be encodable and within limits.
chunk.encode("utf-16-le")
assert utf16_len(chunk) <= 120 + 20, (
f"Chunk {i} UTF-16 length {utf16_len(chunk)} exceeds limit"
)


class TestProxyKwargsForAiohttp:
"""Verify proxy_kwargs_for_aiohttp routes all schemes through ProxyConnector."""
Expand Down