diff --git a/agent/adaptive_context.py b/agent/adaptive_context.py new file mode 100644 index 0000000000000..88078cb055fdc --- /dev/null +++ b/agent/adaptive_context.py @@ -0,0 +1,94 @@ +"""Track response.model changes across LLM calls so the context +compressor can rebudget to the actual backend's context length. + +Many OpenAI-compatible endpoints expose router-style model ids such as +``openrouter/auto`` or ``:free``-suffixed names where the concrete +backend selected per request varies (Llama 3.3, Qwen, DeepSeek, etc.). +Each backend has its own context window. Without observing the live +``response.model`` value, Hermes keeps its compressor calibrated to +whatever the operator configured at startup, which is usually wrong +for whichever backend the router actually picked. + +This module is a tiny state machine: it remembers the last observed +model id and reports back when it changes. It does *not* perform the +``context_length`` lookup (``agent.model_metadata`` already owns that) +and it does *not* mutate the compressor. The caller decides what to do +with the change signal. +""" +from __future__ import annotations + +import logging +import time + +logger = logging.getLogger(__name__) + + +class AdaptiveContextTracker: + """Stateful observer that fires when ``response.model`` changes. + + Typical use:: + + tracker = AdaptiveContextTracker() + new_model = tracker.observe(getattr(response, "model", None)) + if new_model is not None: + ctx = get_model_context_length(new_model, ...) + compressor.update_model(model=new_model, context_length=ctx, ...) + """ + + def __init__(self) -> None: + self._last_seen: str | None = None + self._last_changed_at: float = 0.0 + self._change_count: int = 0 + + def observe(self, response_model: str | None) -> str | None: + """Record the model id reported by an LLM response. + + Returns the new id when it differs from the previously seen + value, so callers know to rebudget. Returns ``None`` on the + first observation, when the value is unchanged, or when the + input is missing/non-string (defensive: response objects in + some adapters may not carry ``model``). + """ + if not response_model or not isinstance(response_model, str): + return None + if self._last_seen is None: + # First observation: adopt silently. The agent was already + # configured with some model id at startup; this is the + # baseline against which subsequent changes are detected. + self._last_seen = response_model + return None + if response_model == self._last_seen: + return None + previous = self._last_seen + self._last_seen = response_model + self._last_changed_at = time.monotonic() + self._change_count += 1 + logger.info( + "adaptive-context: backend changed %r -> %r (change #%d)", + previous, response_model, self._change_count, + ) + return response_model + + @property + def last_seen(self) -> str | None: + return self._last_seen + + @property + def change_count(self) -> int: + return self._change_count + + def summary(self) -> dict: + """Snapshot of tracker state for UX surfaces (e.g. /usage). + + Returns a plain dict so callers don't have to reach into + private attributes. ``seconds_since_last_change`` is ``None`` + until the first transition is observed. + """ + seconds_since: float | None = None + if self._change_count > 0 and self._last_changed_at > 0: + seconds_since = max(0.0, time.monotonic() - self._last_changed_at) + return { + "last_seen": self._last_seen, + "change_count": self._change_count, + "seconds_since_last_change": seconds_since, + } diff --git a/agent/agent_init.py b/agent/agent_init.py index 678db5705cd35..cb50357e8df71 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -1834,6 +1834,24 @@ def init_agent( except Exception: agent.lmstudio_load_mode = "explicit" + # Adaptive context window: observe response.model after each LLM + # call and re-budget the compressor when the upstream router + # resolves to a different backend (e.g. openrouter/auto picking + # Llama-3.3-70B one call and Qwen-2.5 the next, each with a + # different context_length). Off by default. + _compression_cfg_for_adapt = _agent_cfg.get("compression", {}) or {} + agent._adaptive_context_enabled = bool( + _compression_cfg_for_adapt.get("adaptive_context_window", False) + ) + agent._adaptive_context = None + if agent._adaptive_context_enabled: + try: + from agent.adaptive_context import AdaptiveContextTracker + agent._adaptive_context = AdaptiveContextTracker() + except Exception as _ace_err: + logger.debug("adaptive_context_window: init failed (%s)", _ace_err) + agent._adaptive_context_enabled = False + try: agent._tool_guardrails = ToolCallGuardrailController( ToolCallGuardrailConfig.from_mapping( diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index f6f36d1903e97..fb24cca9058c5 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -3281,7 +3281,36 @@ def _perform_api_call(next_api_kwargs): # Log response with provider info if available resp_model = getattr(response, 'model', 'N/A') if response else 'N/A' logging.debug(f"API Response received - Model: {resp_model}, Usage: {response.usage if hasattr(response, 'usage') else 'N/A'}") - + + # Adaptive context window: if the router picked a + # different backend this call, rebudget the compressor + # to the new backend's context_length. + if getattr(agent, "_adaptive_context", None) is not None and response is not None: + try: + _new_model = agent._adaptive_context.observe( + getattr(response, "model", None) + ) + if _new_model: + from agent.model_metadata import get_model_context_length + _new_ctx = get_model_context_length( + _new_model, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + ) + _cc = getattr(agent, "context_compressor", None) + if _cc is not None and _new_ctx: + _cc.update_model( + model=_new_model, + context_length=_new_ctx, + base_url=agent.base_url, + api_key=getattr(agent, "api_key", ""), + provider=agent.provider, + api_mode=agent.api_mode, + ) + except Exception as _ace_loop: + logger.debug("adaptive-context observe failed: %s", _ace_loop) + # Validate response shape before proceeding response_invalid = False error_details = [] diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 1a8021ff984d2..411760b8a38a6 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -697,6 +697,17 @@ compression: # To pin a specific model/provider for compression summaries, use the # auxiliary section below (auxiliary.compression.provider / model). + # When the configured model is a router id (e.g. "openrouter/auto", + # ":free"-suffixed names, model fallback chains), the actual backend + # selected per request varies, and so does its context window. + # If true, Hermes reads response.model after each LLM call and, when + # the backend changes, recomputes context budgets (threshold, tail, + # summary cap) against the new backend's real context_length. + # Default: false. Costs at most one model_metadata lookup per backend + # transition; lookups are cached on disk so subsequent transitions to + # the same backend are free. + adaptive_context_window: false + # ============================================================================= # Tool-result budget (optional) # ============================================================================= diff --git a/cli.py b/cli.py index 2600a5845ccfa..08d113dec9d93 100644 --- a/cli.py +++ b/cli.py @@ -13781,6 +13781,25 @@ def _show_usage(self): print(f" Current context: {last_prompt:,} / {ctx_len:,} ({pct:.0f}%)") print(f" Messages: {msg_count}") print(f" Compressions: {compressions}") + # Adaptive context window: surface live router-tracking state when + # the feature is enabled. The compressor's context_length above + # already reflects any rebudget, so this line is purely informative + # ("which backend is currently driving that number"). + _adapt = getattr(agent, "_adaptive_context", None) + if _adapt is not None: + _summary = _adapt.summary() + if _summary["last_seen"]: + if _summary["change_count"] == 0: + print(f" Adaptive ctx: enabled - baseline: {_summary['last_seen']}") + else: + _changes = _summary["change_count"] + _plural = "" if _changes == 1 else "s" + print( + f" Adaptive ctx: enabled - last seen: {_summary['last_seen']} " + f"({_changes} backend change{_plural})" + ) + else: + print(" Adaptive ctx: enabled (no responses observed yet)") # Account limits -- fetched off-thread with a hard timeout so slow # provider APIs don't hang the prompt. diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index 001d6b7469c4b..b22ea9613228a 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -5540,6 +5540,26 @@ async def _handle_usage_command(self, event: MessageEvent) -> str: lines.append("") lines.extend(breakdown_lines) + # Adaptive context window: surface live router-tracking state + # when the feature is enabled (config flag wired in run_agent). + _adapt = getattr(agent, "_adaptive_context", None) + if _adapt is not None: + _summary = _adapt.summary() + if _summary["last_seen"]: + if _summary["change_count"] == 0: + lines.append(t( + "gateway.usage.label_adaptive_context_baseline", + model=_summary["last_seen"], + )) + else: + lines.append(t( + "gateway.usage.label_adaptive_context_changes", + model=_summary["last_seen"], + count=_summary["change_count"], + )) + else: + lines.append(t("gateway.usage.label_adaptive_context_pending")) + if account_lines: lines.append("") lines.extend(account_lines) diff --git a/locales/en.yaml b/locales/en.yaml index a636a23c27084..e6ba9e2a07acf 100644 --- a/locales/en.yaml +++ b/locales/en.yaml @@ -413,6 +413,9 @@ gateway: breakdown_cat_subagent_definitions: "Subagent definitions" breakdown_cat_memory: "Memory" breakdown_cat_conversation: "Conversation" + label_adaptive_context_pending: "Adaptive ctx: enabled (no responses observed yet)" + label_adaptive_context_baseline: "Adaptive ctx: enabled - baseline: `{model}`" + label_adaptive_context_changes: "Adaptive ctx: enabled - last seen: `{model}` ({count} backend changes)" header_session_info: "📊 **Session Info**" label_messages: "Messages: {count}" label_estimated_context: "Estimated context: ~{count} tokens" diff --git a/tests/agent/test_adaptive_context.py b/tests/agent/test_adaptive_context.py new file mode 100644 index 0000000000000..eef210376b495 --- /dev/null +++ b/tests/agent/test_adaptive_context.py @@ -0,0 +1,111 @@ +"""Unit tests for agent.adaptive_context.AdaptiveContextTracker. + +The tracker is intentionally minimal - it observes model ids and +reports the new id only on change. These tests pin the boundary +behaviours so the run_agent integration can rely on them. +""" +from __future__ import annotations + +import pytest + +from agent.adaptive_context import AdaptiveContextTracker + + +def test_first_observation_returns_none_and_records_baseline(): + t = AdaptiveContextTracker() + assert t.observe("meta-llama/llama-3.3-70b-instruct:free") is None + assert t.last_seen == "meta-llama/llama-3.3-70b-instruct:free" + assert t.change_count == 0 + + +def test_same_model_returns_none(): + t = AdaptiveContextTracker() + t.observe("model-a") + assert t.observe("model-a") is None + assert t.observe("model-a") is None + assert t.change_count == 0 + + +def test_change_returns_new_model_and_increments(): + t = AdaptiveContextTracker() + t.observe("model-a") + new = t.observe("model-b") + assert new == "model-b" + assert t.last_seen == "model-b" + assert t.change_count == 1 + + +def test_multiple_changes_increment_counter(): + t = AdaptiveContextTracker() + t.observe("a") + assert t.observe("b") == "b" + assert t.observe("c") == "c" + assert t.observe("c") is None + assert t.observe("d") == "d" + assert t.change_count == 3 + + +def test_router_to_concrete_backend_transition_fires_once(): + # Realistic scenario: first call returns a concrete backend, second + # call returns a *different* concrete backend (the router picked + # someone else). The tracker should fire on the second. + t = AdaptiveContextTracker() + assert t.observe("meta-llama/llama-3.3-70b-instruct:free") is None + fired = t.observe("qwen/qwen-2.5-72b-instruct:free") + assert fired == "qwen/qwen-2.5-72b-instruct:free" + assert t.change_count == 1 + + +@pytest.mark.parametrize("bad", [None, "", 0, False, 42, [], {}, object()]) +def test_invalid_inputs_return_none_without_changing_state(bad): + t = AdaptiveContextTracker() + t.observe("model-a") # seed + snapshot_last = t.last_seen + snapshot_count = t.change_count + assert t.observe(bad) is None + assert t.last_seen == snapshot_last + assert t.change_count == snapshot_count + + +def test_invalid_input_as_first_observation_does_not_seed(): + t = AdaptiveContextTracker() + assert t.observe(None) is None + assert t.observe("") is None + assert t.last_seen is None + # A subsequent valid observation should still be treated as the baseline + assert t.observe("first-real") is None + assert t.last_seen == "first-real" + assert t.change_count == 0 + + +def test_summary_fresh_tracker(): + t = AdaptiveContextTracker() + s = t.summary() + assert s == { + "last_seen": None, + "change_count": 0, + "seconds_since_last_change": None, + } + + +def test_summary_after_baseline_only(): + t = AdaptiveContextTracker() + t.observe("model-a") + s = t.summary() + assert s["last_seen"] == "model-a" + assert s["change_count"] == 0 + # No change yet, so no elapsed-since-change figure + assert s["seconds_since_last_change"] is None + + +def test_summary_after_change_includes_elapsed(): + t = AdaptiveContextTracker() + t.observe("model-a") + t.observe("model-b") + s = t.summary() + assert s["last_seen"] == "model-b" + assert s["change_count"] == 1 + assert isinstance(s["seconds_since_last_change"], float) + assert s["seconds_since_last_change"] >= 0.0 + # Sanity: a brand-new change should be sub-second + assert s["seconds_since_last_change"] < 5.0