fix(gateway): suppress routine pre-API compression chatter on chat platforms

agent/conversation_compression.py

@@ -71,6 +71,58 @@ def _emit_compaction_done(agent: Any) -> None:
         logger.debug("status_callback error in compaction completion", exc_info=True)
 
 
+# ── Routine compression status templates ────────────────────────────────────
+# Every ROUTINE (non-failure, non-manual-/compress) compression status line the
+# agent emits lives here so the gateway noise filter and its tests can couple
+# to the real emitted wording instead of hand-copied literals. These are
+# suppressed on human-facing chat platforms by _TELEGRAM_NOISY_STATUS_RE
+# (gateway/run.py) — when rewording ANY of them, update that regex and the
+# pinned data in tests/gateway/test_telegram_noise_filter.py in the same PR.
+# Failure notices (⚠ Compression aborted / empty transcript / codex compaction
+# failed) and manual /compress feedback (manual_compression_feedback.py) are
+# deliberate carve-outs from silence and must NOT be added here.
+PRE_API_COMPRESSION_STATUS_TEMPLATE = (
+    "📦 Pre-API compression: ~{tokens:,} tokens "
+    "near the context/output limit. Compacting before the next model call."
+)
+PREFLIGHT_COMPRESSION_STATUS_TEMPLATE = (
+    "📦 Preflight compression: ~{tokens:,} tokens "
+    ">= {threshold:,} threshold. This may take a moment."
+)
+IDLE_COMPACTION_STATUS_TEMPLATE = (
+    "💤 Resumed after {idle_seconds}s idle — compacting "
+    "~{tokens:,} tokens before continuing."
+)
+COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE = (
+    "🗜️ Context too large (~{tokens:,} tokens) — compressing ({attempt}/{cap})..."
+)
+COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE = (
+    "🗜️ Compressed {before} → {after} messages, retrying..."
+)
+COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE = (
+    "🗜️ Compressed ~{before:,} → ~{after:,} tokens, retrying..."
+)
+COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE = (
+    "🗜️ Context reduced to {new_ctx:,} tokens (was {old_ctx:,}), retrying..."
+)
+
+# Sample-formatted instances of every routine compression status line, for
+# behavioral tests that iterate the ACTUAL emitted wording (formatted from the
+# same constants the emission sites use) through the gateway noise filter.
+ROUTINE_COMPRESSION_STATUS_SAMPLES = (
+    COMPACTION_STATUS,
+    PRE_API_COMPRESSION_STATUS_TEMPLATE.format(tokens=123456),
+    PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(tokens=120000, threshold=100000),
+    IDLE_COMPACTION_STATUS_TEMPLATE.format(idle_seconds=3600, tokens=120000),
+    COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE.format(tokens=250000, attempt=1, cap=3),
+    COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE.format(before=30, after=12),
+    COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE.format(before=250000, after=120000),
+    COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE.format(
+        new_ctx=120000, old_ctx=250000
+    ),
+)
+
+
 def _builtin_memory_prompt_snapshot(agent: Any) -> Optional[Tuple[str, str]]:
     """Return the built-in memory text that can affect a system prompt.
 

agent/conversation_loop.py

@@ -28,7 +28,14 @@
 from typing import Any, Dict, List, Optional
 
 from agent.codex_responses_adapter import _summarize_user_message_for_log
-from agent.conversation_compression import conversation_history_after_compression
+from agent.conversation_compression import (
+    COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE,
+    COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE,
+    COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE,
+    COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE,
+    PRE_API_COMPRESSION_STATUS_TEMPLATE,
+    conversation_history_after_compression,
+)
 from agent.display import KawaiiSpinner
 from agent.error_classifier import FailoverReason, classify_api_error
 from agent.iteration_budget import IterationBudget
@@ -1279,8 +1286,9 @@ def run_conversation(
                 max_compression_attempts,
             )
             agent._emit_status(
-                f"📦 Pre-API compression: ~{request_pressure_tokens:,} tokens "
-                f"near the context/output limit. Compacting before the next model call."
+                PRE_API_COMPRESSION_STATUS_TEMPLATE.format(
+                    tokens=request_pressure_tokens
+                )
             )
             _last_preflight_pressure = request_pressure_tokens
             messages, active_system_prompt = agent._compress_context(
@@ -3494,8 +3502,9 @@ def _perform_api_call(next_api_kwargs):
                         )
                         if len(messages) < original_len or old_ctx > _reduced_ctx:
                             agent._buffer_status(
-                                f"🗜️ Context reduced to {_reduced_ctx:,} tokens "
-                                f"(was {old_ctx:,}), retrying..."
+                                COMPRESSION_RETRY_CONTEXT_REDUCED_STATUS_TEMPLATE.format(
+                                    new_ctx=_reduced_ctx, old_ctx=old_ctx
+                                )
                             )
                             time.sleep(2)
                             _retry.restart_with_compressed_messages = True
@@ -3756,9 +3765,9 @@ def _perform_api_call(next_api_kwargs):
 
                     if len(messages) < original_len or (new_tokens > 0 and new_tokens < original_tokens * 0.95):
                         if len(messages) < original_len:
-                            agent._buffer_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...")
+                            agent._buffer_status(COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE.format(before=original_len, after=len(messages)))
                         else:
-                            agent._buffer_status(f"🗜️ Compressed ~{original_tokens:,} → ~{new_tokens:,} tokens, retrying...")
+                            agent._buffer_status(COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE.format(before=original_tokens, after=new_tokens))
                         time.sleep(2)  # Brief pause between compression retries
                         _retry.restart_with_compressed_messages = True
                         break
@@ -3976,7 +3985,7 @@ def _perform_api_call(next_api_kwargs):
                             "failed": True,
                             "compression_exhausted": True,
                         }
-                    agent._buffer_status(f"🗜️ Context too large (~{approx_tokens:,} tokens) — compressing ({compression_attempts}/{max_compression_attempts})...")
+                    agent._buffer_status(COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE.format(tokens=approx_tokens, attempt=compression_attempts, cap=max_compression_attempts))
 
                     original_len = len(messages)
                     original_tokens = estimate_messages_tokens_rough(messages)
@@ -3997,9 +4006,9 @@ def _perform_api_call(next_api_kwargs):
 
                     if len(messages) < original_len or (new_tokens > 0 and new_tokens < original_tokens * 0.95) or (new_ctx and new_ctx < old_ctx):
                         if len(messages) < original_len:
-                            agent._buffer_status(f"🗜️ Compressed {original_len} → {len(messages)} messages, retrying...")
+                            agent._buffer_status(COMPRESSION_RETRY_MESSAGES_STATUS_TEMPLATE.format(before=original_len, after=len(messages)))
                         elif new_tokens > 0 and new_tokens < original_tokens * 0.95:
-                            agent._buffer_status(f"🗜️ Compressed ~{original_tokens:,} → ~{new_tokens:,} tokens, retrying...")
+                            agent._buffer_status(COMPRESSION_RETRY_TOKENS_STATUS_TEMPLATE.format(before=original_tokens, after=new_tokens))
                         time.sleep(2)  # Brief pause between compression retries
                         _retry.restart_with_compressed_messages = True
                         break

agent/turn_context.py

@@ -31,7 +31,11 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Mapping, Optional
 
-from agent.conversation_compression import conversation_history_after_compression
+from agent.conversation_compression import (
+    IDLE_COMPACTION_STATUS_TEMPLATE,
+    PREFLIGHT_COMPRESSION_STATUS_TEMPLATE,
+    conversation_history_after_compression,
+)
 from agent.iteration_budget import IterationBudget
 from agent.memory_manager import build_memory_context_block
 from agent.model_metadata import (
@@ -659,8 +663,9 @@ def build_turn_context(
                     agent.session_id or "none",
                 )
                 agent._emit_status(
-                    f"💤 Resumed after {int(_idle_gap)}s idle — compacting "
-                    f"~{_idle_tokens:,} tokens before continuing."
+                    IDLE_COMPACTION_STATUS_TEMPLATE.format(
+                        idle_seconds=int(_idle_gap), tokens=_idle_tokens
+                    )
                 )
                 _idle_input = messages
                 messages, active_system_prompt = agent._compress_context(
@@ -767,9 +772,10 @@ def build_turn_context(
                 f"{_compressor.context_length:,}",
             )
             agent._emit_status(
-                f"📦 Preflight compression: ~{_preflight_tokens:,} tokens "
-                f">= {_compressor.threshold_tokens:,} threshold. "
-                "This may take a moment."
+                PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(
+                    tokens=_preflight_tokens,
+                    threshold=_compressor.threshold_tokens,
+                )
             )
             # Preflight passes honor the same configured per-turn cap
             # (compression.max_attempts) as the loop's compression sites;

contributors/emails/matt.strawbridge@lotuscollective.ai

@@ -0,0 +1 @@
+matt-strawbridge

gateway/run.py

@@ -79,9 +79,23 @@
     r"|configured\s+compression\s+model\s+.+\s+failed"
     r"|no\s+auxiliary\s+llm\s+provider\s+configured"
     r"|auto-lowered\s+compression\s+threshold"
+    # #69332 reworded the auto-lower notice to "Auto-lowered this session's
+    # threshold to N tokens" — keep both generations covered.
+    r"|auto-lowered\s+(?:this\s+)?session'?s?\s+threshold"
+    r"|configured\s+auxiliary\s+compression\s+provider\s+.+\s+unavailable"
+    r"|skipping\s+concurrent\s+compression"
     r"|compacting\s+context\s+[—-]\s+summarizing\s+earlier\s+conversation"
     r"|resumed\s+after\s+\d+s\s+idle\s+[—-]\s+compacting"
     r"|preflight\s+compression"
+    r"|pre[- ]api\s+compression"
+    # Buffered attempt/overflow retry chatter replayed through _emit_status
+    # when a turn exhausts retries. The ", retrying"/"— compressing" anchors
+    # keep manual /compress feedback ("Compressed: 30 → 12 messages") and
+    # failure notices out of the match.
+    r"|context\s+too\s+large\s+\(~[\d,]+\s+tokens\)\s+[—-]+\s+compressing"
+    r"|compressed\s+\d[\d,]*\s+(?:→|->)\s+\d[\d,]*\s+messages,\s+retrying"
+    r"|compressed\s+~[\d,]+\s+(?:→|->)\s+~[\d,]+\s+tokens,\s+retrying"
+    r"|context\s+reduced\s+to\s+[\d,]+\s+tokens\s+\(was\s+[\d,]+\),\s+retrying"
     r"|session\s+compressed\s+\d+\s+times"
     r"|rate\s+limited\.\s+waiting\s+\d"
     r"|retrying\s+in\s+\d"

tests/gateway/test_telegram_noise_filter.py

@@ -2,6 +2,7 @@
 
 import pytest
 
+from agent.conversation_compression import ROUTINE_COMPRESSION_STATUS_SAMPLES
 from gateway.config import Platform
 from gateway.run import (
     _prepare_gateway_status_message,
@@ -30,12 +31,57 @@
 
 NOISY_STATUS_MESSAGES = [
     "🗜️ Preflight compression check before sending...",
+    (
+        "📦 Pre-API compression: ~123,456 tokens near the context/output limit. "
+        "Compacting before the next model call."
+    ),
     "🗜️ Compacting context — summarizing earlier conversation so I can continue...",
     "💤 Resumed after 3600s idle — compacting ~120,000 tokens before continuing.",
     "⚠️  Session compressed 12 times — accuracy may degrade. Consider /new to start fresh.",
     "⚠ Compression summary failed: upstream error. Inserted a fallback context marker.",
     "⏱️ Rate limited. Waiting 30.0s (attempt 2/3)...",
     "⏳ Retrying in 4.2s (attempt 1/3)...",
+    # Buffered overflow/attempt-cap retry chatter (replayed on retry exhaustion).
+    "🗜️ Context too large (~250,000 tokens) — compressing (1/3)...",
+    "🗜️ Compressed 30 → 12 messages, retrying...",
+    "🗜️ Compressed ~250,000 → ~120,000 tokens, retrying...",
+    "🗜️ Context reduced to 120,000 tokens (was 250,000), retrying...",
+    # Post-#69332 auto-lower wording + aux-provider/lock chatter.
+    (
+        "⚠ Compression model small (openrouter) context is 32,000 tokens, but "
+        "the main model big (anthropic)'s compression threshold was 100,000 "
+        "tokens. Auto-lowered this session's threshold to 30,000 tokens so "
+        "compression can run."
+    ),
+    (
+        "⚠ Configured auxiliary compression provider 'openai' is unavailable — "
+        "context compression will drop middle turns without a summary. Check "
+        "auxiliary.compression in config.yaml and reauthenticate that provider."
+    ),
+    (
+        "⚠ Skipping concurrent compression — another path is already "
+        "compressing this session. Will retry after it finishes."
+    ),
+]
+
+# Messages that must NEVER be swallowed by the compression-noise filter:
+# deliberate carve-outs from routine-compression silence — manual /compress
+# feedback (manual_compression_feedback.py headlines) and abort/failure
+# notices that require user action.
+VISIBLE_COMPRESSION_MESSAGES = [
+    "Compressed: 30 → 12 messages",
+    "Compression aborted: 30 messages preserved",
+    "Compressed with fallback: 30 → 12 messages",
+    "No changes from compression: 30 messages",
+    (
+        "⚠ Compression aborted: auth failure. No messages were dropped — "
+        "conversation continues unchanged. Run /compress to retry, or /new "
+        "to start a fresh session."
+    ),
+    (
+        "⚠ Compression returned an empty transcript. No session split was "
+        "performed; conversation continues unchanged."
+    ),
 ]
 
 
@@ -69,13 +115,49 @@ def test_programmatic_surfaces_keep_raw_status():
         )
 
 
+@pytest.mark.parametrize("message", ["still on it", "⏳ Working — 3 min"])
+def test_telegram_status_keeps_legitimate_heartbeat_messages(message):
+    """The compression filter must not swallow user-facing work heartbeats."""
+    assert _prepare_gateway_status_message(Platform.TELEGRAM, "lifecycle", message) == message
+
+
 @pytest.mark.parametrize("platform", CHAT_PLATFORMS)
 @pytest.mark.parametrize("message", NOISY_STATUS_MESSAGES)
 def test_all_chat_gateways_suppress_noise(platform, message):
     """Operational lifecycle/retry noise must be suppressed on every chat surface."""
     assert _prepare_gateway_status_message(platform, "warn", message) is None
 
 
+@pytest.mark.parametrize("platform", CHAT_PLATFORMS)
+@pytest.mark.parametrize(
+    "message", ROUTINE_COMPRESSION_STATUS_SAMPLES, ids=lambda m: m[:32]
+)
+def test_all_routine_compression_statuses_suppressed_from_source_constants(
+    platform, message
+):
+    """Every ROUTINE compression status the agent actually emits is filtered.
+
+    Iterates the sample-formatted status strings built from the SAME
+    constants the emission sites use (agent/conversation_compression.py's
+    ROUTINE_COMPRESSION_STATUS_SAMPLES), so a reworded emit site that drifts
+    past the noise regex fails here without anyone remembering to re-copy
+    the literal into this file.
+    """
+    assert _prepare_gateway_status_message(platform, "lifecycle", message) is None
+
+
+@pytest.mark.parametrize("platform", CHAT_PLATFORMS)
+@pytest.mark.parametrize("message", VISIBLE_COMPRESSION_MESSAGES, ids=lambda m: m[:32])
+def test_manual_compress_feedback_and_failure_notices_stay_visible(platform, message):
+    """Manual /compress feedback and abort notices must never be swallowed.
+
+    These are the deliberate carve-outs from routine-compression silence
+    (#16775 failures, manual_compression_feedback.py) — widening the noise
+    regex must not start eating them.
+    """
+    assert _prepare_gateway_status_message(platform, "warn", message) == message
+
+
 @pytest.mark.parametrize("platform", ["whatsapp", "slack", "signal", "matrix"])
 def test_chat_gateways_redact_secret_in_provider_error(platform):
     """Provider-error bodies carrying secrets must never reach chat users.