From 1c1a86e6fd06cb7587becdbb4248e7b7098fc26e Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:08:52 -0700 Subject: [PATCH 1/3] fix(auto-router): accept every reminder marker pair a harness emits reminder_markers held one (open, close) pair, so a harness that wraps injected context differently per agent type only got the slice of traffic using the configured envelope stripped. Every other agent type kept hitting the original bug: its reminder-only turn never stripped to empty, won "newest human ask", and the harness blob got classified in place of the real question, choosing the tier and therefore the spend. The field now takes a list of ReminderMarkerPair, following the KeywordTierRule pattern already in this file so each pair validates itself and errors point at reminder_markers.N.close rather than a bare index. Blocks from different pairs can nest, which the gap construction could not handle: resuming the kept text at an inner block's end walks back inside the enclosing block and leaks its remainder. Running the block ends through a maximum collapses nested and overlapping spans without a separate merge pass, and stays linear in block count, which a fold over a growing tuple of merged spans would not. A single pair's ends already increase, so the maximum is the identity and the default path is byte-identical: verified against the shipped function over 200k generated inputs, and every existing reminder test passes unchanged. The prior single-pair config shape is rejected loudly at startup and at /model/new rather than silently stripping nothing. --- .../complexity_router/__init__.py | 2 + .../complexity_router/complexity_router.py | 71 ++++-- .../complexity_router/config.py | 47 ++-- .../router_strategy/test_complexity_router.py | 218 +++++++++++++++++- 4 files changed, 290 insertions(+), 48 deletions(-) diff --git a/litellm/router_strategy/complexity_router/__init__.py b/litellm/router_strategy/complexity_router/__init__.py index 1830ff506e94..aa618cc807e2 100644 --- a/litellm/router_strategy/complexity_router/__init__.py +++ b/litellm/router_strategy/complexity_router/__init__.py @@ -16,6 +16,7 @@ DEFAULT_COMPLEXITY_CONFIG, ComplexityRouterConfig, ComplexityTier, + ReminderMarkerPair, ) __all__ = [ @@ -24,5 +25,6 @@ "ComplexityRouter", "ComplexityRouterConfig", "ComplexityTier", + "ReminderMarkerPair", "classification_system_prompt", ] diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7bbe01191e32..a69509fc37a5 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -19,7 +19,7 @@ import random import re from collections.abc import Iterator, Mapping, Sequence -from itertools import islice +from itertools import accumulate, islice from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast @@ -233,6 +233,7 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" +_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) _TRUNCATION_MARKER: Final = "..." @@ -253,10 +254,8 @@ def _message_text(content: object) -> str: return content if isinstance(content, str) else "" -def _reminder_block_spans( - lowered: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE -) -> Iterator[tuple[int, int]]: - """Span of each complete reminder block, left to right. +def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]: + """Span of each complete reminder block for one marker pair, left to right. Literal `str.find`, not a regex: the delimiters are fixed strings, and `.*?` retried its lazy quantifier from every opening tag, so repeated unclosed tags were quadratic @@ -272,17 +271,36 @@ def _reminder_block_spans( yield start, cursor -def _strip_reminder_blocks(text: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str: - """Remove every complete reminder block from text, keeping everything written around them.""" - spans: Final = tuple(_reminder_block_spans(text.lower(), open_marker, close_marker)) +def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str: + """Remove every complete reminder block from text, keeping everything written around them. + + Blocks from different pairs can nest or overlap, which the gap construction below would + otherwise mishandle: an inner block's end would resume the kept text partway through the outer + block, leaking the rest of that block into the classified ask. Running the block ends through a + maximum resumes each gap past the furthest block seen so far, which collapses nested and + overlapping spans without a separate merge pass. A single pair's ends already increase, so the + maximum is the identity there and the default path is byte-identical to a plain scan. + + Deliberately linear in both the text and the block count. This runs pre-routing on input any + keyholder controls, and both a regex scan and a fold that rebuilds a growing tuple of merged + spans go quadratic on inputs that are cheap to send. + """ + lowered: Final = text.lower() + spans: Final = tuple( + sorted( + span + for open_marker, close_marker in marker_pairs + for span in _reminder_block_spans(lowered, open_marker, close_marker) + ) + ) if not spans: return text.strip() - keep_from: Final = (0, *(end for _, end in spans)) + keep_from: Final = (0, *accumulate((end for _, end in spans), max)) keep_to: Final = (*(start for start, _ in spans), len(text)) return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip())) -def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str: +def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str: """Message content as the text a human wrote, with complete reminder blocks removed. Harnesses inject reminders as ordinary text alongside the live ask, so the block is stripped and @@ -291,18 +309,18 @@ def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker one, and this same string drives escalation keywords and keyword_tier_rules, which choose the model and therefore the spend. An unclosed tag is not a block and is left intact. """ - return _strip_reminder_blocks(_message_text(content), open_marker, close_marker) + return _strip_reminder_blocks(_message_text(content), marker_pairs) def _iter_human_asks_newest_first( - messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE) + messages: Sequence[Mapping[str, object]], + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> Iterator[str]: """Yield user-turn texts that carry a real human ask, newest first, with harness noise removed.""" - open_marker, close_marker = markers return ( text for msg in reversed(messages) - if msg.get("role") == "user" and (text := _human_text(msg.get("content"), open_marker, close_marker)) + if msg.get("role") == "user" and (text := _human_text(msg.get("content"), marker_pairs)) ) @@ -341,7 +359,8 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) def _newest_turn_ask( - messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE) + messages: Sequence[Mapping[str, object]], + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -352,12 +371,12 @@ def _newest_turn_ask( newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None) if newest_user_turn is None: return None - return _human_text(newest_user_turn.get("content"), *markers) or None + return _human_text(newest_user_turn.get("content"), marker_pairs) or None def _extract_current_ask_and_system_prompt( messages: Sequence[Mapping[str, object]], - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> tuple[str | None, str | None]: """The last real human ask and the last system prompt; either is None if absent. @@ -365,7 +384,7 @@ def _extract_current_ask_and_system_prompt( the caller routes to its default model. That is the correct answer rather than a gap to fill: filling it would hand tier selection to harness-injected text. """ - current_ask: Final = next(_iter_human_asks_newest_first(messages, markers), None) + current_ask: Final = next(_iter_human_asks_newest_first(messages, marker_pairs), None) system_prompt: Final = next( ( text @@ -385,7 +404,7 @@ def _truncate(text: str, limit: int) -> str: def _iter_context_turns_newest_first( messages: Sequence[Mapping[str, object]], include_assistant: bool, - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> Iterator[tuple[str, str]]: """Yield (role, text) for turns eligible as classifier context, newest first. @@ -401,7 +420,7 @@ def _iter_context_turns_newest_first( for msg in reversed(messages) if isinstance(role := msg.get("role"), str) and role in roles - and (text := _human_text(msg.get("content"), *markers)) + and (text := _human_text(msg.get("content"), marker_pairs)) ) @@ -411,7 +430,7 @@ def _extract_prior_turns( window_size: int, per_turn_chars: int, include_assistant: bool, - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> tuple[tuple[str, str], ...]: """Up to window_size turns other than current_ask, oldest first, as (role, text). @@ -431,7 +450,7 @@ def _extract_prior_turns( prior: Final = islice( ( turn - for turn in _iter_context_turns_newest_first(messages, include_assistant, markers) + for turn in _iter_context_turns_newest_first(messages, include_assistant, marker_pairs) if turn[1] != current_ask ), window_size, @@ -556,7 +575,11 @@ def __init__( if self.config.escalation_keywords is not None else DEFAULT_ESCALATION_KEYWORDS ) - self._reminder_markers: tuple[str, str] = self.config.reminder_markers or (_REMINDER_OPEN, _REMINDER_CLOSE) + self._reminder_markers: tuple[tuple[str, str], ...] = ( + tuple((pair.open, pair.close) for pair in self.config.reminder_markers) + if self.config.reminder_markers + else _DEFAULT_REMINDER_MARKERS + ) # Lazily built on first semantic request and cached for reuse (route # embeddings are static, only the prompt is embedded per request). The lock @@ -993,7 +1016,7 @@ async def _classify_with_llm( window_size=self.config.classifier_context_window_size, per_turn_chars=self.config.classifier_context_per_turn_chars, include_assistant=include_assistant, - markers=self._reminder_markers, + marker_pairs=self._reminder_markers, ) if context_enabled else () diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index f9d3bd9ae67f..69609a973b02 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -59,6 +59,30 @@ def _normalize_keywords(self) -> "KeywordTierRule": return self +class ReminderMarkerPair(BaseModel): + """One open/close delimiter pair a harness wraps injected context in. + + Normalizing here rather than at the scan is what makes matching case-insensitive: markers reach + the scan already lowered, so it lowercases only the haystack and never the needles. Stripping + keeps YAML indentation whitespace from becoming part of the delimiter. + """ + + open: str = Field(description="Opening delimiter, e.g. ''") + close: str = Field(description="Closing delimiter, e.g. ''") + + @model_validator(mode="after") + def _normalize(self) -> "ReminderMarkerPair": + open_marker: Final = self.open.strip().lower() + close_marker: Final = self.close.strip().lower() + if not open_marker or not close_marker: + raise ValueError("reminder_markers entries must not be blank") + if open_marker == close_marker: + raise ValueError("reminder_markers open and close must be different strings") + self.open = open_marker + self.close = close_marker + return self + + # ─── Default Keyword Lists ─── # Note: Keywords should be full words/phrases to avoid substring false positives. # The matching logic uses word boundary detection for single-word keywords. @@ -498,12 +522,15 @@ class ComplexityRouterConfig(BaseModel): description="RoutingPlugin instances that narrow the classified tier's candidate models before selection", ) - reminder_markers: tuple[str, str] | None = Field( + reminder_markers: tuple[ReminderMarkerPair, ...] | None = Field( default=None, + min_length=1, description=( - "Override the (open, close) marker pair used to recognize and strip harness-injected " - "reminder blocks before classification. Defaults to Claude Code's convention, " - "('', ''), when unset. Matching is case-insensitive." + "Override the delimiter pairs used to recognize and strip harness-injected reminder " + "blocks before classification. A harness that wraps injected context differently per " + "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " + "adds to, the built-in default of ('', ''), so a " + "harness that also emits that pair lists it too. Matching is case-insensitive." ), ) @@ -601,18 +628,6 @@ def _validate_plugins_adaptive_combo(self) -> "ComplexityRouterConfig": ) return self - @model_validator(mode="after") - def _normalize_reminder_markers(self) -> "ComplexityRouterConfig": - if self.reminder_markers is None: - return self - open_marker, close_marker = (marker.strip().lower() for marker in self.reminder_markers) - if not open_marker or not close_marker: - raise ValueError("reminder_markers entries must not be blank") - if open_marker == close_marker: - raise ValueError("reminder_markers open and close must be different strings") - self.reminder_markers = (open_marker, close_marker) - return self - def tier_label(self, tier: ComplexityTier) -> str: """Operator-facing display name for a tier, falling back to its canonical name.""" return self.tier_labels.get(tier, "").strip() or tier.value diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index b3f1e9297413..9c323647eecf 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -112,6 +112,52 @@ def test_init_with_config(self, mock_router_instance, basic_config): assert router.config.tiers["SIMPLE"] == "gpt-4o-mini" assert router.config.tiers["REASONING"] == "o1-preview" + def test_configured_marker_pairs_reach_the_ask_extraction(self, mock_router_instance, basic_config): + """Marker pairs configured in YAML must actually reach the code that strips them. + + The config field, the validator and the scan were each covered on their own, but nothing + exercised config.reminder_markers -> self._reminder_markers, so the router could have parsed + a valid config and still classified on unstripped text. Asserting through the extraction the + router feeds its classifier is what makes that wiring a regression rather than a silent gap. + """ + from litellm.router_strategy.complexity_router.complexity_router import ( + _extract_current_ask_and_system_prompt, + ) + + ask = "Derive the amortized complexity of a splay tree access" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "reminder_markers": [ + {"open": "<<>>", "close": "<<>>"}, + {"open": "[[SUBAGENT_BEGIN]]", "close": "[[SUBAGENT_END]]"}, + ], + }, + ) + + assert router._reminder_markers == ( + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ) + messages = [ + {"role": "user", "content": ask}, + {"role": "assistant", "content": "Working on it."}, + {"role": "user", "content": "[[SUBAGENT_BEGIN]]Budget: 42 tokens remaining.[[SUBAGENT_END]]"}, + ] + assert _extract_current_ask_and_system_prompt(messages, router._reminder_markers)[0] == ask + + def test_unconfigured_marker_pairs_fall_back_to_the_builtin_default(self, mock_router_instance, basic_config): + """A config that never mentions reminder_markers keeps stripping .""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=basic_config, + ) + + assert router._reminder_markers == (("", ""),) + def test_init_without_config(self, mock_router_instance): """Test initialization without configuration uses defaults.""" router = ComplexityRouter( @@ -2991,17 +3037,68 @@ def test_reminder_markers_unset_defaults_to_none(self): def test_reminder_markers_are_normalized(self): """Markers are stripped and lowercased, matching how the built-in constants are compared.""" config = ComplexityRouterConfig( - reminder_markers=(" <<>> ", "<<>>"), + reminder_markers=[{"open": " <<>> ", "close": "<<>>"}], + ) + assert config.reminder_markers is not None + assert (config.reminder_markers[0].open, config.reminder_markers[0].close) == ( + "<<>>", + "<<>>", ) - assert config.reminder_markers == ("<<>>", "<<>>") + + def test_reminder_markers_keep_every_configured_pair_in_order(self): + """Every pair a harness emits survives validation, not just the first.""" + config = ComplexityRouterConfig( + reminder_markers=[ + {"open": "<<>>", "close": "<<>>"}, + {"open": "[[SUBAGENT_BEGIN]]", "close": "[[SUBAGENT_END]]"}, + {"open": "%%CRON_BEGIN%%", "close": "%%CRON_END%%"}, + ], + ) + assert config.reminder_markers is not None + assert [(pair.open, pair.close) for pair in config.reminder_markers] == [ + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ("%%cron_begin%%", "%%cron_end%%"), + ] def test_reminder_markers_reject_blank_entry(self): with pytest.raises(ValidationError, match="must not be blank"): - ComplexityRouterConfig(reminder_markers=("", "<<>>")) + ComplexityRouterConfig(reminder_markers=[{"open": "", "close": "<<>>"}]) def test_reminder_markers_reject_identical_open_and_close(self): with pytest.raises(ValidationError, match="must be different"): - ComplexityRouterConfig(reminder_markers=("<<>>", "<<>>")) + ComplexityRouterConfig(reminder_markers=[{"open": "<<>>", "close": "<<>>"}]) + + def test_reminder_markers_reject_a_bad_pair_anywhere_in_the_list(self): + """Validation runs per pair, so a broken entry after a good one is still caught.""" + with pytest.raises(ValidationError, match="must be different"): + ComplexityRouterConfig( + reminder_markers=[ + {"open": "<<>>", "close": "<<>>"}, + {"open": "<<>>", "close": "<<>>"}, + ], + ) + + def test_reminder_markers_reject_empty_list(self): + """An explicitly empty list is ambiguous, so it fails loudly instead of silently defaulting. + + Left to fall through, an empty list resolves to the built-in pair, which + reads as "strip nothing" in the config and does the opposite. Matching on the length error + keeps this from passing for some unrelated reason if the field type changes. + """ + with pytest.raises(ValidationError, match="at least 1 item"): + ComplexityRouterConfig(reminder_markers=[]) + + def test_reminder_markers_reject_the_old_flat_pair_form(self): + """The pre-list shape is rejected loudly rather than silently routing on unstripped text. + + reminder_markers took a bare (open, close) string pair before it took a list of pairs. A + config still using that shape must fail validation at startup and at /model/new write time, + because the alternative -- accepting it and stripping nothing -- hands tier selection, and + therefore spend, to harness-injected text without any signal that it happened. + """ + with pytest.raises(ValidationError, match="valid dictionary or instance of ReminderMarkerPair"): + ComplexityRouterConfig(reminder_markers=("", "")) class _StubEncoder: @@ -4306,7 +4403,6 @@ async def test_reasoning_override_is_its_own_cause(self, complexity_router): # The score is still recorded, but the cause is what says it did not decide. assert decision["score"] < decision["tier_boundaries"]["complex_reasoning"] - @pytest.mark.asyncio async def test_an_unrenamed_router_writes_no_tier_label(self, complexity_router): """Renaming is opt-in, so a deployment that never renamed must gain no new key. @@ -4919,12 +5015,73 @@ def test_custom_markers_skip_a_reminder_only_follow_up_message(self): """ from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt - markers = ("<<>>", "<<>>") - follow_up_reminder = f"{markers[0]}Budget: 42 tokens remaining. Do not mention this.{markers[1]}" + pair = ("<<>>", "<<>>") + follow_up_reminder = f"{pair[0]}Budget: 42 tokens remaining. Do not mention this.{pair[1]}" messages = [_ASKED, _ANSWERED, {"role": "user", "content": follow_up_reminder}] assert _extract_current_ask_and_system_prompt(messages)[0] == follow_up_reminder - assert _extract_current_ask_and_system_prompt(messages, markers)[0] == _ASK + assert _extract_current_ask_and_system_prompt(messages, (pair,))[0] == _ASK + + def test_every_configured_marker_pair_is_stripped_not_just_the_first(self): + """One deployment serves a harness whose agent types each use a different envelope. + + Main agent, subagent and cron wrap injected context in different open/close pairs, and they + all route through the same auto-router. When only one pair could be configured, the other + agent types kept hitting the original bug: their reminder-only turn never stripped to empty, + won "newest human ask", and the harness blob got classified in place of the real question. + Each pair in turn must be skipped, so this fails if only the first configured pair is used. + """ + from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt + + pairs = ( + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ("%%cron_begin%%", "%%cron_end%%"), + ) + for open_marker, close_marker in pairs: + reminder_only_turn = f"{open_marker}Budget: 42 tokens remaining.{close_marker}" + messages = [_ASKED, _ANSWERED, {"role": "user", "content": reminder_only_turn}] + + assert _extract_current_ask_and_system_prompt(messages, pairs)[0] == _ASK, open_marker + + def test_a_block_nested_inside_another_pairs_block_does_not_leak(self): + """Nested blocks from two pairs must strip whole, not resume inside the outer block. + + Spans are collected per pair and can nest. Resuming the kept text at each block's own end + walks backwards into the enclosing block, so the outer block's remainder (and its dangling + close marker) survive into the classified ask. That is harness text choosing the tier, and + therefore the spend. Overlapping and disjoint spans strip correctly either way, so this + nested case is what pins the behavior. + """ + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + nested = "<<>>budget[[subagent_begin]]inner[[subagent_end]]do not mention<<>>" + + assert _strip_reminder_blocks(f"{nested} what is a splay tree?", pairs) == "what is a splay tree?" + + def test_overlapping_blocks_from_two_pairs_strip_whole(self): + """Interleaved (not nested) blocks still strip everything they jointly cover.""" + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + overlapping = "<<>>a[[subagent_begin]]b<<>>c[[subagent_end]]" + + assert _strip_reminder_blocks(f"{overlapping} what is a splay tree?", pairs) == "what is a splay tree?" + + def test_an_unclosed_marker_in_one_pair_does_not_suppress_another_pairs_blocks(self): + """Each pair scans independently, so one pair's dangling opener is not a global stop. + + An unclosed tag ends that pair's scan by design and is left intact as prose. It must not + also swallow a different pair's complete block, which would put harness text back in front + of the classifier. + """ + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + text = "<<>> why is [[subagent_begin]]noise[[subagent_end]] my tag stripped?" + + assert _strip_reminder_blocks(text, pairs) == "<<>> why is my tag stripped?" @pytest.mark.parametrize( "messages,current_ask,window,per_turn_chars,include_assistant,expected", @@ -5084,6 +5241,28 @@ def test_prior_turn_window(self, messages, current_ask, window, per_turn_chars, assert _extract_prior_turns(messages, current_ask, window, per_turn_chars, include_assistant) == expected + def test_prior_turn_context_strips_every_configured_pair(self): + """The classifier's context window is stripped with the same pairs as the ask. + + Prior turns are quoted verbatim into the LLM classifier payload, so a pair that is honored + when picking the ask but ignored when building context puts the harness blob back in front + of the classifier through the other door. This covers the _extract_prior_turns call the ask + extraction tests never reach. + """ + from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + messages = [ + {"role": "user", "content": "[[subagent_begin]]budget blob[[subagent_end]]what about b-trees?"}, + {"role": "user", "content": "<<>>other blob<<>>and heaps?"}, + {"role": "user", "content": "current ask"}, + ] + + assert _extract_prior_turns(messages, "current ask", 5, 200, False, pairs) == ( + ("user", "what about b-trees?"), + ("user", "and heaps?"), + ) + def test_reminder_scan_is_linear_on_adversarial_input(self): """Unclosed reminder tags must not make stripping superlinear. @@ -5105,6 +5284,29 @@ def test_reminder_scan_is_linear_on_adversarial_input(self): assert elapsed < 1.0, f"stripping {len(adversarial)} chars took {elapsed:.2f}s; scan is not linear" assert result == adversarial + def test_reminder_scan_stays_linear_in_block_count_across_pairs(self): + """Many *complete* blocks across several pairs must not go quadratic either. + + Collapsing nested and overlapping spans is required for correctness once more than one pair + is configured, and the obvious way to write it -- folding merged spans into a growing tuple + -- is quadratic in block count. Unlike the unclosed-tag case above, these blocks all close, + so they actually produce spans. This input is a few hundred KB, which any keyholder can send + pre-routing, and it fails loudly if the collapse is ever rewritten as a fold. + """ + import time + + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("", ""), ("", "")) + adversarial = "xy" * 25_000 + + start = time.perf_counter() + result = _strip_reminder_blocks(f"{adversarial} what is a splay tree?", pairs) + elapsed = time.perf_counter() - start + + assert elapsed < 1.0, f"stripping {50_000} blocks took {elapsed:.2f}s; collapse is not linear" + assert result == "what is a splay tree?" + @pytest.mark.asyncio async def test_llm_classifier_includes_prior_turns_context(self, llm_complexity_router, mock_router_instance): """Test that the LLM classifier receives prior-turn context in the user message.""" From df95a4c24f34a515acae92da8c483914134e2a70 Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:10:29 -0700 Subject: [PATCH 2/3] docs(auto-router): document reminder_markers in the complexity router README --- .../complexity_router/README.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index b1fdb0044be8..259933dbb9ea 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -171,6 +171,27 @@ If 2+ reasoning markers are detected in the user message, the request is automat Reasoning markers in the system prompt do **not** trigger the reasoning override. This prevents system prompts like "Think step by step before answering" from forcing all requests to the reasoning tier. +### Harness Reminder Blocks + +Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead + +By default a block is anything between `` and ``. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + reminder_markers: + - open: "<<>>" + close: "<<>>" + - open: "[[SUBAGENT_CONTEXT_BEGIN]]" + close: "[[SUBAGENT_CONTEXT_END]]" +``` + +Setting `reminder_markers` replaces the built-in `` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten + ### Code Detection Technical code keywords are detected case-insensitively and include: From 1907f05707429d56f7398dc86d4a026ed1756cef Mon Sep 17 00:00:00 2001 From: Abhimanyu Kapur <38531241+akapur99@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:50:58 -0700 Subject: [PATCH 3/3] chore(ui): regenerate dashboard API types for the reminder_markers shape --- .../router_strategy/test_complexity_router.py | 4 ++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 27 +++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index 9c323647eecf..8e9e32f58984 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -5963,7 +5963,9 @@ def test_unset_system_prompt_defaults_to_none(self): @pytest.mark.asyncio async def test_custom_prompt_is_sent_verbatim_as_the_system_role(self, mock_router_instance, llm_classifier_config): - custom = "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + custom = ( + "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + ) router = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 65434407f74e..959c99218962 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31560,6 +31560,26 @@ export interface components { /** Review Notes */ review_notes?: string | null; }; + /** + * ReminderMarkerPair + * @description One open/close delimiter pair a harness wraps injected context in. + * + * Normalizing here rather than at the scan is what makes matching case-insensitive: markers reach + * the scan already lowered, so it lowercases only the haystack and never the needles. Stripping + * keeps YAML indentation whitespace from becoming part of the delimiter. + */ + ReminderMarkerPair: { + /** + * Close + * @description Closing delimiter, e.g. '' + */ + close: string; + /** + * Open + * @description Opening delimiter, e.g. '' + */ + open: string; + }; /** * RequestComplexityRouterConfig * @description The part of a complexity-router config a request can carry. @@ -31672,12 +31692,9 @@ export interface components { reasoning_keywords?: string[] | null; /** * Reminder Markers - * @description Override the (open, close) marker pair used to recognize and strip harness-injected reminder blocks before classification. Defaults to Claude Code's convention, ('', ''), when unset. Matching is case-insensitive. + * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in default of ('', ''), so a harness that also emits that pair lists it too. Matching is case-insensitive. */ - reminder_markers?: [ - string, - string - ] | null; + reminder_markers?: components["schemas"]["ReminderMarkerPair"][] | null; /** * Return Raw Model Name * @description Return the resolved raw model name in the response model field instead of the client-requested complexity-router alias