diff --git a/agent/agent_init.py b/agent/agent_init.py index ea756dc398d9..c268c37d505c 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1839,6 +1839,18 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: } else: compression_model_thresholds = {} + # Absolute token cap: when set, compression triggers at the lower of + # the ratio-based threshold and this absolute count. Clamped to the + # model's context length at apply-time so a cap above the window is + # a no-op (ratio-based threshold wins). + compression_threshold_tokens = _compression_cfg.get("threshold_tokens") + if compression_threshold_tokens is not None: + try: + compression_threshold_tokens = int(compression_threshold_tokens) + if compression_threshold_tokens <= 0: + compression_threshold_tokens = None + except (TypeError, ValueError): + compression_threshold_tokens = None # In-place compaction: when True, compress_context() rewrites the message # list + rebuilds the system prompt WITHOUT rotating the session id (no # parent_session_id chain, no `name #N` renumber). See #38763 and @@ -2271,6 +2283,7 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: abort_on_summary_failure=compression_abort_on_summary_failure, max_tokens=agent.max_tokens, model_thresholds=compression_model_thresholds, + threshold_tokens_cap=compression_threshold_tokens, ) _bind_session_state = getattr(agent.context_compressor, "bind_session_state", None) if callable(_bind_session_state): @@ -2482,7 +2495,11 @@ def _moa_reference_relay(event: str, **kwargs: Any) -> None: _active_threshold_pct = getattr( agent.context_compressor, "threshold_percent", compression_threshold ) - print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,})") + _cap_note = "" + _cap = getattr(agent.context_compressor, "threshold_tokens_cap", None) + if _cap and _cap > 0: + _cap_note = f" (capped at {_cap:,} tokens)" + print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (compress at {int(_active_threshold_pct*100)}% = {agent.context_compressor.threshold_tokens:,}{_cap_note})") else: print(f"📊 Context limit: {agent.context_compressor.context_length:,} tokens (auto-compression disabled)") # Notice with the exact opt-back-out command. Printed inline at startup diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 48228ef2b720..3d76960619d6 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -1231,6 +1231,11 @@ def update_model( self.threshold_tokens = self._compute_threshold_tokens( context_length, self.threshold_percent, self.max_tokens, ) + # Re-apply the absolute token cap so it survives model switches + # and fallback activations. The cap is a first-class config value + # stored on the compressor instance, not a one-time post-construction + # patch — this is why update_model() must re-apply it. + self._apply_threshold_tokens_cap() # Recalculate token budgets for the new context length so the # compressor stays calibrated after a model switch (e.g. 200K → 32K). target_tokens = int(self.threshold_tokens * self.summary_target_ratio) @@ -1293,6 +1298,36 @@ def _coerce_max_tokens(value: Any) -> int | None: return None return ivalue if ivalue > 0 else None + @staticmethod + def _coerce_threshold_tokens_cap(value: Any) -> int | None: + """Normalize a threshold_tokens cap to a positive int or None. + + None means "no absolute cap — use the ratio-based threshold only". + Non-numeric or non-positive values are treated as None so a bad + config value never silently caps the threshold at zero. + """ + if value is None: + return None + try: + ivalue = int(value) + except (TypeError, ValueError): + return None + return ivalue if ivalue > 0 else None + + def _apply_threshold_tokens_cap(self) -> None: + """Apply the absolute token cap if configured. + + After ``threshold_tokens`` is (re)computed from the ratio-based + percent, clamp it to the cap so compression never fires later + than the user's preferred absolute token count. The cap itself + is clamped to the current context length so a cap larger than + the model's window is a no-op (the ratio-based threshold wins). + """ + if self.threshold_tokens_cap is not None and self.threshold_tokens_cap > 0: + _effective_cap = min(self.threshold_tokens_cap, self.context_length) + if _effective_cap < self.threshold_tokens: + self.threshold_tokens = _effective_cap + @staticmethod def _effective_threshold_percent( context_length: int, threshold_percent: float, @@ -1368,6 +1403,7 @@ def __init__( abort_on_summary_failure: bool = False, max_tokens: int | None = None, model_thresholds: dict[str, float] | None = None, + threshold_tokens_cap: Any = None, ): self.model = model self.base_url = base_url @@ -1387,6 +1423,14 @@ def __init__( model, self.model_thresholds, threshold_percent, ) self.threshold_percent = self._base_threshold_percent + # Absolute token cap from config (compression.threshold_tokens). When + # set, the effective trigger point is min(ratio-based threshold, cap) + # so compression never fires later than the user's preferred token + # count regardless of which model is active. Applied in __init__ and + # re-applied in update_model() so it survives model switches/fallbacks. + self.threshold_tokens_cap = self._coerce_threshold_tokens_cap( + threshold_tokens_cap, + ) self.protect_first_n = protect_first_n self.protect_last_n = protect_last_n self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80)) @@ -1432,6 +1476,9 @@ def __init__( self.threshold_tokens = self._compute_threshold_tokens( self.context_length, threshold_percent, self.max_tokens, ) + # Apply absolute token cap (compression.threshold_tokens) — takes + # the lower of the ratio-based threshold and the cap. + self._apply_threshold_tokens_cap() self.compression_count = 0 # Derive token budgets: ratio is relative to the threshold, not total context diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 4a66d502c028..c93d4db15a36 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -424,6 +424,15 @@ compression: # "claude-sonnet": 0.35 # "gpt-5": 0.30 + # Optional absolute token cap for the compression trigger (default: null = disabled). + # When set, compression fires at the LOWER of the ratio-based threshold and this + # absolute token count — first-fires-wins. It never fires later than this count + # regardless of which model is active (useful when switching between models with + # very different context windows). Clamped to the model's context length at + # apply-time, so a cap above the window is a no-op (ratio-based threshold wins). + # Survives model switches and fallback activations. + # threshold_tokens: 200000 + # Existing Codex gpt-5.5 behavior: raise Hermes' compaction trigger to 85% # for the ChatGPT Codex OAuth route. Set false to opt back down to threshold. codex_gpt55_autoraise: true diff --git a/contributors/emails/maly.dan@gmail.com b/contributors/emails/maly.dan@gmail.com new file mode 100644 index 000000000000..a7fc6da71741 --- /dev/null +++ b/contributors/emails/maly.dan@gmail.com @@ -0,0 +1 @@ +DanielMaly diff --git a/gateway/run.py b/gateway/run.py index 354962ee627d..0444f5a34111 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -17644,6 +17644,7 @@ async def _run_process_watcher(self, watcher: dict) -> None: ("compression", "enabled"), ("compression", "threshold"), ("compression", "model_thresholds"), + ("compression", "threshold_tokens"), ("compression", "codex_gpt55_autoraise"), ("compression", "codex_app_server_auto"), ("compression", "target_ratio"), diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 59d702ac73dd..3d49b238b11f 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1469,6 +1469,10 @@ def _ensure_hermes_home_managed(home: Path): # floored at 0.75 (raise-only) so compaction # doesn't fire with half the window still free; # set this above 0.75 to override the floor. + "threshold_tokens": None, # absolute token cap — when set, compression + # triggers at the lower of the ratio-based + # threshold and this token count. Clamped to + # the model's context length at apply-time. "target_ratio": 0.20, # fraction of threshold to preserve as recent tail "protect_last_n": 20, # minimum recent messages to keep uncompressed "max_attempts": 3, # compression retry rounds before a turn gives up @@ -8548,6 +8552,14 @@ def show_config(): print(f" Enabled: {'yes' if enabled else 'no'}") if enabled: print(f" Threshold: {compression.get('threshold', 0.50) * 100:.0f}%") + _tt = compression.get('threshold_tokens') + if _tt is not None: + try: + _tt = int(_tt) + if _tt > 0: + print(f" Token cap: {_tt:,} tokens (takes lower of ratio vs absolute)") + except (TypeError, ValueError): + pass print(f" Target ratio: {compression.get('target_ratio', 0.20) * 100:.0f}% of threshold preserved") print(f" Protect last: {compression.get('protect_last_n', 20)} messages") print(f" Protect first: {compression.get('protect_first_n', 3)} non-system head messages") diff --git a/tests/agent/test_context_compressor.py b/tests/agent/test_context_compressor.py index 15a44b34c856..f0db2a36b402 100644 --- a/tests/agent/test_context_compressor.py +++ b/tests/agent/test_context_compressor.py @@ -3165,6 +3165,207 @@ def test_summary_failure_cooldown_survives_same_runtime_refresh(self): assert comp._summary_failure_cooldown_until == cooldown_until +class TestThresholdTokensCap: + """Tests for the absolute token cap (compression.threshold_tokens). + + The cap takes the lower of the ratio-based threshold and the absolute + count. It must survive model switches (update_model re-applies it) + and be clamped to the model's context length. + """ + + def test_cap_lower_than_ratio_uses_cap(self): + """When the cap is lower than the ratio-based threshold, the cap wins.""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=50_000, + ) + # Ratio-based: 200000 * 0.50 = 100000. Cap: 50000. Effective: 50000. + assert comp.threshold_tokens == 50_000 + + def test_cap_higher_than_ratio_uses_ratio(self): + """When the cap is higher than the ratio-based threshold, the ratio wins.""" + with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=2_000_000, + ) + # Ratio-based: 1000000 * 0.50 = 500000. Cap: 2000000, clamped to 1000000. + # Effective: min(500000, 1000000) = 500000. + assert comp.threshold_tokens == 500_000 + + def test_no_cap_uses_ratio_only(self): + """Without a cap, the ratio-based threshold is used.""" + with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + ) + assert comp.threshold_tokens == 500_000 + assert comp.threshold_tokens_cap is None + + def test_cap_survives_model_switch(self): + """The cap must be re-applied after update_model() switches to a + different context length. This is the core sweeper feedback: the + old PR's post-construction patch was undone by update_model() + restoring _configured_threshold_percent.""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=40_000, + ) + assert comp.threshold_tokens == 40_000 # cap wins on 200K model + + # Switch to a 100K model — ratio-based would be 50000, but cap is 40000 + comp.update_model("model-b", context_length=100_000) + assert comp.threshold_tokens == 40_000 # cap still wins + + def test_cap_survives_model_switch_to_smaller_window(self): + """When switching to a model whose ratio-based threshold is below + the cap, the ratio-based threshold wins (cap is a ceiling, not a floor).""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=50_000, + ) + assert comp.threshold_tokens == 50_000 # cap wins on 200K (ratio=100K) + + # Switch to a 64K model — ratio-based floor is 64000 (MINIMUM_CONTEXT_LENGTH) + # which is > 50000 cap, so... actually 64000 > 50000 means cap still wins + # Let's test with a 80K model: ratio=40000, cap=50000 → ratio wins + comp.update_model("model-b", context_length=80_000) + assert comp.threshold_tokens <= 50_000 # cap is a ceiling + # 80000 * 0.50 = 40000, floored to 64000, cap 50000 → min(64000, 50000) = 50000 + # The floor raises it to 64000, then cap clamps to 50000 + assert comp.threshold_tokens == 50_000 + + def test_cap_clamped_to_context_length(self): + """A cap larger than the context length is clamped, so the + ratio-based threshold wins for small-context models.""" + with patch("agent.context_compressor.get_model_context_length", return_value=64_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=500_000, + ) + # 64000 * 0.50 = 32000, floored to 64000 (MINIMUM_CONTEXT_LENGTH), + # degenerate: floored >= window → 85% of 64000 = 54400. + # Cap 500000 clamped to 64000. min(54400, 64000) = 54400. + assert comp.threshold_tokens == 54400 # ratio-based wins + + def test_cap_with_max_tokens_reservation(self): + """The cap applies after max_tokens reservation is factored in.""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + max_tokens=32_768, + threshold_tokens_cap=50_000, + ) + # effective_window = 200000 - 32768 = 167232 + # ratio: 167232 * 0.50 = 83616, floored to max(83616, 64000) = 83616 + # cap: min(50000, 200000) = 50000. min(83616, 50000) = 50000. + assert comp.threshold_tokens == 50_000 + + def test_cap_survives_model_switch_with_max_tokens(self): + """The cap survives model switch even when max_tokens changes.""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + max_tokens=32_768, + threshold_tokens_cap=50_000, + ) + assert comp.threshold_tokens == 50_000 + + # Switch to a smaller model with different max_tokens + comp.update_model("model-b", context_length=100_000, max_tokens=16_384) + # effective_window = 100000 - 16384 = 83616 + # ratio: 83616 * 0.50 = 41808, floored to max(41808, 64000) = 64000 + # degenerate: floored (64000) >= effective_window (83616)? No, 64000 < 83616. + # So threshold = 64000. cap: min(50000, 100000) = 50000. min(64000, 50000) = 50000. + assert comp.threshold_tokens == 50_000 + + def test_invalid_cap_treated_as_none(self): + """Non-numeric, zero, or negative cap values are treated as None.""" + with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000): + comp0 = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=0, + ) + assert comp0.threshold_tokens_cap is None + assert comp0.threshold_tokens == 500_000 + + comp_neg = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=-100, + ) + assert comp_neg.threshold_tokens_cap is None + + comp_str = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap="not-a-number", + ) + assert comp_str.threshold_tokens_cap is None + + def test_should_compress_fires_at_cap_below_ratio_threshold(self): + """Behavioral: with a cap below the ratio-based threshold, + should_compress() fires once usage crosses the cap — even though + the percentage threshold has not been reached (first-fires-wins).""" + with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=200_000, + ) + # Ratio-based would be 500K; cap pulls the trigger down to 200K. + assert comp.should_compress(150_000) is False # below cap + assert comp.should_compress(200_000) is True # at cap (below 500K pct) + assert comp.should_compress(250_000) is True # above cap + + def test_default_config_disabled_and_no_behavior_change(self): + """DEFAULT_CONFIG ships threshold_tokens=None (disabled) and both + None and 0 leave the ratio-based trigger byte-identical.""" + from hermes_cli.config import DEFAULT_CONFIG + assert DEFAULT_CONFIG["compression"]["threshold_tokens"] is None + + with patch("agent.context_compressor.get_model_context_length", return_value=1_000_000): + baseline = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + ) + comp_none = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=None, + ) + comp_zero = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=0, + ) + assert comp_none.threshold_tokens == baseline.threshold_tokens + assert comp_zero.threshold_tokens == baseline.threshold_tokens + # And after a model switch, still identical to baseline. + baseline.update_model("model-b", context_length=200_000) + comp_none.update_model("model-b", context_length=200_000) + comp_zero.update_model("model-b", context_length=200_000) + assert comp_none.threshold_tokens == baseline.threshold_tokens + assert comp_zero.threshold_tokens == baseline.threshold_tokens + + def test_pct_floor_unaffected_by_cap(self): + """The small-context pct floor (raise-only to 0.75 under 512K) is + computed independently of the cap: the cap clamps the resulting + token threshold but never changes threshold_percent, and a + cap-free small-context model keeps the floored pct.""" + with patch("agent.context_compressor.get_model_context_length", return_value=200_000): + comp = ContextCompressor( + "model-a", threshold_percent=0.50, quiet_mode=True, + threshold_tokens_cap=100_000, + ) + # Floor raised pct to 0.75 (200K < 512K) regardless of the cap. + assert comp.threshold_percent == 0.75 + # Cap clamps the token trigger below the floored pct value (150K). + assert comp.threshold_tokens == 100_000 + # Switching to a large-context model drops the pct back to the + # configured 0.50 — cap presence doesn't perturb the re-derivation. + comp.update_model("model-b", context_length=1_000_000) + assert comp.threshold_percent == 0.50 + assert comp.threshold_tokens == 100_000 # cap still wins over 500K + + class TestTruncateToolCallArgsJson: """Regression tests for #11762. diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index e04e1eb30e26..b1973865dfab 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -744,6 +744,7 @@ All compression settings live in `config.yaml` (no environment variables). compression: enabled: true # Toggle compression on/off threshold: 0.50 # Compress at this % of context limit + threshold_tokens: null # Absolute token cap (optional) — takes lower of ratio vs absolute target_ratio: 0.20 # Fraction of threshold to preserve as recent tail protect_last_n: 20 # Min recent messages to keep uncompressed protect_first_n: 3 # Non-system head messages pinned across compactions (0 = pin nothing) @@ -765,6 +766,8 @@ Older configs with `compression.summary_model`, `compression.summary_provider`, `protect_first_n` controls how many **non-system** head messages are pinned across every compaction. Default `3` — the opening user/assistant exchange survives every summarizer pass so the original goal stays visible. On long-running rolling-compaction sessions where the opening turn is no longer relevant, set `protect_first_n: 0` to pin nothing but the system prompt + summary + tail. The system prompt itself is always preserved regardless of this setting. +`threshold_tokens` sets an optional **absolute token cap** for the compression trigger. When set, compression fires at the lower of the ratio-based `threshold` and this absolute count — so compression never fires later than the user's preferred token number regardless of which model is active. This solves the problem where switching between models with different context windows (e.g. 1M → 400K) shifts the absolute trigger point. The cap is clamped to the model's context length, so setting it higher than the model supports is safe — the ratio-based threshold is used instead. Default `null` (disabled — ratio-based threshold only). The cap survives model switches and fallback activations. + :::tip Gateway hot-reload of compression and context length As of recent releases, editing `model.context_length` or any `compression.*` key in `config.yaml` on a running gateway takes effect on the next message — no gateway restart, no `/reset`, no session rotation required. The cached-agent signature includes these keys, so the gateway transparently rebuilds the agent when it sees a change. API keys and tool/skill config still require the usual reload paths. :::