feat(agent): re-budget context compressor when a router swaps the backend

agent/adaptive_context.py

@@ -0,0 +1,94 @@
+"""Track response.model changes across LLM calls so the context
+compressor can rebudget to the actual backend's context length.
+
+Many OpenAI-compatible endpoints expose router-style model ids such as
+``openrouter/auto`` or ``:free``-suffixed names where the concrete
+backend selected per request varies (Llama 3.3, Qwen, DeepSeek, etc.).
+Each backend has its own context window. Without observing the live
+``response.model`` value, Hermes keeps its compressor calibrated to
+whatever the operator configured at startup, which is usually wrong
+for whichever backend the router actually picked.
+
+This module is a tiny state machine: it remembers the last observed
+model id and reports back when it changes. It does *not* perform the
+``context_length`` lookup (``agent.model_metadata`` already owns that)
+and it does *not* mutate the compressor. The caller decides what to do
+with the change signal.
+"""
+from __future__ import annotations
+
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class AdaptiveContextTracker:
+    """Stateful observer that fires when ``response.model`` changes.
+
+    Typical use::
+
+        tracker = AdaptiveContextTracker()
+        new_model = tracker.observe(getattr(response, "model", None))
+        if new_model is not None:
+            ctx = get_model_context_length(new_model, ...)
+            compressor.update_model(model=new_model, context_length=ctx, ...)
+    """
+
+    def __init__(self) -> None:
+        self._last_seen: str | None = None
+        self._last_changed_at: float = 0.0
+        self._change_count: int = 0
+
+    def observe(self, response_model: str | None) -> str | None:
+        """Record the model id reported by an LLM response.
+
+        Returns the new id when it differs from the previously seen
+        value, so callers know to rebudget. Returns ``None`` on the
+        first observation, when the value is unchanged, or when the
+        input is missing/non-string (defensive: response objects in
+        some adapters may not carry ``model``).
+        """
+        if not response_model or not isinstance(response_model, str):
+            return None
+        if self._last_seen is None:
+            # First observation: adopt silently. The agent was already
+            # configured with some model id at startup; this is the
+            # baseline against which subsequent changes are detected.
+            self._last_seen = response_model
+            return None
+        if response_model == self._last_seen:
+            return None
+        previous = self._last_seen
+        self._last_seen = response_model
+        self._last_changed_at = time.monotonic()
+        self._change_count += 1
+        logger.info(
+            "adaptive-context: backend changed %r -> %r (change #%d)",
+            previous, response_model, self._change_count,
+        )
+        return response_model
+
+    @property
+    def last_seen(self) -> str | None:
+        return self._last_seen
+
+    @property
+    def change_count(self) -> int:
+        return self._change_count
+
+    def summary(self) -> dict:
+        """Snapshot of tracker state for UX surfaces (e.g. /usage).
+
+        Returns a plain dict so callers don't have to reach into
+        private attributes. ``seconds_since_last_change`` is ``None``
+        until the first transition is observed.
+        """
+        seconds_since: float | None = None
+        if self._change_count > 0 and self._last_changed_at > 0:
+            seconds_since = max(0.0, time.monotonic() - self._last_changed_at)
+        return {
+            "last_seen": self._last_seen,
+            "change_count": self._change_count,
+            "seconds_since_last_change": seconds_since,
+        }

agent/agent_init.py

@@ -1834,6 +1834,24 @@ def init_agent(
     except Exception:
         agent.lmstudio_load_mode = "explicit"
 
+    # Adaptive context window: observe response.model after each LLM
+    # call and re-budget the compressor when the upstream router
+    # resolves to a different backend (e.g. openrouter/auto picking
+    # Llama-3.3-70B one call and Qwen-2.5 the next, each with a
+    # different context_length). Off by default.
+    _compression_cfg_for_adapt = _agent_cfg.get("compression", {}) or {}
+    agent._adaptive_context_enabled = bool(
+        _compression_cfg_for_adapt.get("adaptive_context_window", False)
+    )
+    agent._adaptive_context = None
+    if agent._adaptive_context_enabled:
+        try:
+            from agent.adaptive_context import AdaptiveContextTracker
+            agent._adaptive_context = AdaptiveContextTracker()
+        except Exception as _ace_err:
+            logger.debug("adaptive_context_window: init failed (%s)", _ace_err)
+            agent._adaptive_context_enabled = False
+
     try:
         agent._tool_guardrails = ToolCallGuardrailController(
             ToolCallGuardrailConfig.from_mapping(

agent/conversation_loop.py

@@ -3281,7 +3281,36 @@ def _perform_api_call(next_api_kwargs):
                     # Log response with provider info if available
                     resp_model = getattr(response, 'model', 'N/A') if response else 'N/A'
                     logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}")
-
+
+                # Adaptive context window: if the router picked a
+                # different backend this call, rebudget the compressor
+                # to the new backend's context_length.
+                if getattr(agent, "_adaptive_context", None) is not None and response is not None:
+                    try:
+                        _new_model = agent._adaptive_context.observe(
+                            getattr(response, "model", None)
+                        )
+                        if _new_model:
+                            from agent.model_metadata import get_model_context_length
+                            _new_ctx = get_model_context_length(
+                                _new_model,
+                                base_url=agent.base_url,
+                                api_key=getattr(agent, "api_key", ""),
+                                provider=agent.provider,
+                            )
+                            _cc = getattr(agent, "context_compressor", None)
+                            if _cc is not None and _new_ctx:
+                                _cc.update_model(
+                                    model=_new_model,
+                                    context_length=_new_ctx,
+                                    base_url=agent.base_url,
+                                    api_key=getattr(agent, "api_key", ""),
+                                    provider=agent.provider,
+                                    api_mode=agent.api_mode,
+                                )
+                    except Exception as _ace_loop:
+                        logger.debug("adaptive-context observe failed: %s", _ace_loop)
+
                 # Validate response shape before proceeding
                 response_invalid = False
                 error_details = []

cli-config.yaml.example

@@ -697,6 +697,17 @@ compression:
   # To pin a specific model/provider for compression summaries, use the
   # auxiliary section below (auxiliary.compression.provider / model).
 
+  # When the configured model is a router id (e.g. "openrouter/auto",
+  # ":free"-suffixed names, model fallback chains), the actual backend
+  # selected per request varies, and so does its context window.
+  # If true, Hermes reads response.model after each LLM call and, when
+  # the backend changes, recomputes context budgets (threshold, tail,
+  # summary cap) against the new backend's real context_length.
+  # Default: false. Costs at most one model_metadata lookup per backend
+  # transition; lookups are cached on disk so subsequent transitions to
+  # the same backend are free.
+  adaptive_context_window: false
+
 # =============================================================================
 # Tool-result budget (optional)
 # =============================================================================

cli.py

@@ -13781,6 +13781,25 @@ def _show_usage(self):
         print(f"  Current context:  {last_prompt:,} / {ctx_len:,} ({pct:.0f}%)")
         print(f"  Messages:         {msg_count}")
         print(f"  Compressions:     {compressions}")
+        # Adaptive context window: surface live router-tracking state when
+        # the feature is enabled. The compressor's context_length above
+        # already reflects any rebudget, so this line is purely informative
+        # ("which backend is currently driving that number").
+        _adapt = getattr(agent, "_adaptive_context", None)
+        if _adapt is not None:
+            _summary = _adapt.summary()
+            if _summary["last_seen"]:
+                if _summary["change_count"] == 0:
+                    print(f"  Adaptive ctx:     enabled - baseline: {_summary['last_seen']}")
+                else:
+                    _changes = _summary["change_count"]
+                    _plural = "" if _changes == 1 else "s"
+                    print(
+                        f"  Adaptive ctx:     enabled - last seen: {_summary['last_seen']} "
+                        f"({_changes} backend change{_plural})"
+                    )
+            else:
+                print("  Adaptive ctx:     enabled (no responses observed yet)")
 
         # Account limits -- fetched off-thread with a hard timeout so slow
         # provider APIs don't hang the prompt.

gateway/slash_commands.py

@@ -5540,6 +5540,26 @@ async def _handle_usage_command(self, event: MessageEvent) -> str:
                 lines.append("")
                 lines.extend(breakdown_lines)
 
+            # Adaptive context window: surface live router-tracking state
+            # when the feature is enabled (config flag wired in run_agent).
+            _adapt = getattr(agent, "_adaptive_context", None)
+            if _adapt is not None:
+                _summary = _adapt.summary()
+                if _summary["last_seen"]:
+                    if _summary["change_count"] == 0:
+                        lines.append(t(
+                            "gateway.usage.label_adaptive_context_baseline",
+                            model=_summary["last_seen"],
+                        ))
+                    else:
+                        lines.append(t(
+                            "gateway.usage.label_adaptive_context_changes",
+                            model=_summary["last_seen"],
+                            count=_summary["change_count"],
+                        ))
+                else:
+                    lines.append(t("gateway.usage.label_adaptive_context_pending"))
+
             if account_lines:
                 lines.append("")
                 lines.extend(account_lines)

locales/en.yaml

@@ -413,6 +413,9 @@ gateway:
     breakdown_cat_subagent_definitions: "Subagent definitions"
     breakdown_cat_memory:               "Memory"
     breakdown_cat_conversation:         "Conversation"
+    label_adaptive_context_pending:  "Adaptive ctx: enabled (no responses observed yet)"
+    label_adaptive_context_baseline: "Adaptive ctx: enabled - baseline: `{model}`"
+    label_adaptive_context_changes:  "Adaptive ctx: enabled - last seen: `{model}` ({count} backend changes)"
     header_session_info:      "📊 **Session Info**"
     label_messages:           "Messages: {count}"
     label_estimated_context:  "Estimated context: ~{count} tokens"

tests/agent/test_adaptive_context.py

@@ -0,0 +1,111 @@
+"""Unit tests for agent.adaptive_context.AdaptiveContextTracker.
+
+The tracker is intentionally minimal - it observes model ids and
+reports the new id only on change. These tests pin the boundary
+behaviours so the run_agent integration can rely on them.
+"""
+from __future__ import annotations
+
+import pytest
+
+from agent.adaptive_context import AdaptiveContextTracker
+
+
+def test_first_observation_returns_none_and_records_baseline():
+    t = AdaptiveContextTracker()
+    assert t.observe("meta-llama/llama-3.3-70b-instruct:free") is None
+    assert t.last_seen == "meta-llama/llama-3.3-70b-instruct:free"
+    assert t.change_count == 0
+
+
+def test_same_model_returns_none():
+    t = AdaptiveContextTracker()
+    t.observe("model-a")
+    assert t.observe("model-a") is None
+    assert t.observe("model-a") is None
+    assert t.change_count == 0
+
+
+def test_change_returns_new_model_and_increments():
+    t = AdaptiveContextTracker()
+    t.observe("model-a")
+    new = t.observe("model-b")
+    assert new == "model-b"
+    assert t.last_seen == "model-b"
+    assert t.change_count == 1
+
+
+def test_multiple_changes_increment_counter():
+    t = AdaptiveContextTracker()
+    t.observe("a")
+    assert t.observe("b") == "b"
+    assert t.observe("c") == "c"
+    assert t.observe("c") is None
+    assert t.observe("d") == "d"
+    assert t.change_count == 3
+
+
+def test_router_to_concrete_backend_transition_fires_once():
+    # Realistic scenario: first call returns a concrete backend, second
+    # call returns a *different* concrete backend (the router picked
+    # someone else). The tracker should fire on the second.
+    t = AdaptiveContextTracker()
+    assert t.observe("meta-llama/llama-3.3-70b-instruct:free") is None
+    fired = t.observe("qwen/qwen-2.5-72b-instruct:free")
+    assert fired == "qwen/qwen-2.5-72b-instruct:free"
+    assert t.change_count == 1
+
+
+@pytest.mark.parametrize("bad", [None, "", 0, False, 42, [], {}, object()])
+def test_invalid_inputs_return_none_without_changing_state(bad):
+    t = AdaptiveContextTracker()
+    t.observe("model-a")  # seed
+    snapshot_last = t.last_seen
+    snapshot_count = t.change_count
+    assert t.observe(bad) is None
+    assert t.last_seen == snapshot_last
+    assert t.change_count == snapshot_count
+
+
+def test_invalid_input_as_first_observation_does_not_seed():
+    t = AdaptiveContextTracker()
+    assert t.observe(None) is None
+    assert t.observe("") is None
+    assert t.last_seen is None
+    # A subsequent valid observation should still be treated as the baseline
+    assert t.observe("first-real") is None
+    assert t.last_seen == "first-real"
+    assert t.change_count == 0
+
+
+def test_summary_fresh_tracker():
+    t = AdaptiveContextTracker()
+    s = t.summary()
+    assert s == {
+        "last_seen": None,
+        "change_count": 0,
+        "seconds_since_last_change": None,
+    }
+
+
+def test_summary_after_baseline_only():
+    t = AdaptiveContextTracker()
+    t.observe("model-a")
+    s = t.summary()
+    assert s["last_seen"] == "model-a"
+    assert s["change_count"] == 0
+    # No change yet, so no elapsed-since-change figure
+    assert s["seconds_since_last_change"] is None
+
+
+def test_summary_after_change_includes_elapsed():
+    t = AdaptiveContextTracker()
+    t.observe("model-a")
+    t.observe("model-b")
+    s = t.summary()
+    assert s["last_seen"] == "model-b"
+    assert s["change_count"] == 1
+    assert isinstance(s["seconds_since_last_change"], float)
+    assert s["seconds_since_last_change"] >= 0.0
+    # Sanity: a brand-new change should be sub-second
+    assert s["seconds_since_last_change"] < 5.0