diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md index b1fdb0044be8..259933dbb9ea 100644 --- a/litellm/router_strategy/complexity_router/README.md +++ b/litellm/router_strategy/complexity_router/README.md @@ -171,6 +171,27 @@ If 2+ reasoning markers are detected in the user message, the request is automat Reasoning markers in the system prompt do **not** trigger the reasoning override. This prevents system prompts like "Think step by step before answering" from forcing all requests to the reasoning tier. +### Harness Reminder Blocks + +Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead + +By default a block is anything between `` and ``. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit: + +```yaml +model_list: + - model_name: smart-router + litellm_params: + model: auto_router/complexity_router + complexity_router_config: + reminder_markers: + - open: "<<>>" + close: "<<>>" + - open: "[[SUBAGENT_CONTEXT_BEGIN]]" + close: "[[SUBAGENT_CONTEXT_END]]" +``` + +Setting `reminder_markers` replaces the built-in `` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten + ### Code Detection Technical code keywords are detected case-insensitively and include: diff --git a/litellm/router_strategy/complexity_router/__init__.py b/litellm/router_strategy/complexity_router/__init__.py index 1830ff506e94..aa618cc807e2 100644 --- a/litellm/router_strategy/complexity_router/__init__.py +++ b/litellm/router_strategy/complexity_router/__init__.py @@ -16,6 +16,7 @@ DEFAULT_COMPLEXITY_CONFIG, ComplexityRouterConfig, ComplexityTier, + ReminderMarkerPair, ) __all__ = [ @@ -24,5 +25,6 @@ "ComplexityRouter", "ComplexityRouterConfig", "ComplexityTier", + "ReminderMarkerPair", "classification_system_prompt", ] diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py index 7bbe01191e32..a69509fc37a5 100644 --- a/litellm/router_strategy/complexity_router/complexity_router.py +++ b/litellm/router_strategy/complexity_router/complexity_router.py @@ -19,7 +19,7 @@ import random import re from collections.abc import Iterator, Mapping, Sequence -from itertools import islice +from itertools import accumulate, islice from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast @@ -233,6 +233,7 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None _REMINDER_OPEN: Final = "" _REMINDER_CLOSE: Final = "" +_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),) _TRUNCATION_MARKER: Final = "..." @@ -253,10 +254,8 @@ def _message_text(content: object) -> str: return content if isinstance(content, str) else "" -def _reminder_block_spans( - lowered: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE -) -> Iterator[tuple[int, int]]: - """Span of each complete reminder block, left to right. +def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]: + """Span of each complete reminder block for one marker pair, left to right. Literal `str.find`, not a regex: the delimiters are fixed strings, and `.*?` retried its lazy quantifier from every opening tag, so repeated unclosed tags were quadratic @@ -272,17 +271,36 @@ def _reminder_block_spans( yield start, cursor -def _strip_reminder_blocks(text: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str: - """Remove every complete reminder block from text, keeping everything written around them.""" - spans: Final = tuple(_reminder_block_spans(text.lower(), open_marker, close_marker)) +def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str: + """Remove every complete reminder block from text, keeping everything written around them. + + Blocks from different pairs can nest or overlap, which the gap construction below would + otherwise mishandle: an inner block's end would resume the kept text partway through the outer + block, leaking the rest of that block into the classified ask. Running the block ends through a + maximum resumes each gap past the furthest block seen so far, which collapses nested and + overlapping spans without a separate merge pass. A single pair's ends already increase, so the + maximum is the identity there and the default path is byte-identical to a plain scan. + + Deliberately linear in both the text and the block count. This runs pre-routing on input any + keyholder controls, and both a regex scan and a fold that rebuilds a growing tuple of merged + spans go quadratic on inputs that are cheap to send. + """ + lowered: Final = text.lower() + spans: Final = tuple( + sorted( + span + for open_marker, close_marker in marker_pairs + for span in _reminder_block_spans(lowered, open_marker, close_marker) + ) + ) if not spans: return text.strip() - keep_from: Final = (0, *(end for _, end in spans)) + keep_from: Final = (0, *accumulate((end for _, end in spans), max)) keep_to: Final = (*(start for start, _ in spans), len(text)) return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip())) -def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str: +def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str: """Message content as the text a human wrote, with complete reminder blocks removed. Harnesses inject reminders as ordinary text alongside the live ask, so the block is stripped and @@ -291,18 +309,18 @@ def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker one, and this same string drives escalation keywords and keyword_tier_rules, which choose the model and therefore the spend. An unclosed tag is not a block and is left intact. """ - return _strip_reminder_blocks(_message_text(content), open_marker, close_marker) + return _strip_reminder_blocks(_message_text(content), marker_pairs) def _iter_human_asks_newest_first( - messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE) + messages: Sequence[Mapping[str, object]], + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> Iterator[str]: """Yield user-turn texts that carry a real human ask, newest first, with harness noise removed.""" - open_marker, close_marker = markers return ( text for msg in reversed(messages) - if msg.get("role") == "user" and (text := _human_text(msg.get("content"), open_marker, close_marker)) + if msg.get("role") == "user" and (text := _human_text(msg.get("content"), marker_pairs)) ) @@ -341,7 +359,8 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None) def _newest_turn_ask( - messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE) + messages: Sequence[Mapping[str, object]], + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> str | None: """The human ask on the newest user turn, or None when that turn carries only plumbing. @@ -352,12 +371,12 @@ def _newest_turn_ask( newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None) if newest_user_turn is None: return None - return _human_text(newest_user_turn.get("content"), *markers) or None + return _human_text(newest_user_turn.get("content"), marker_pairs) or None def _extract_current_ask_and_system_prompt( messages: Sequence[Mapping[str, object]], - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> tuple[str | None, str | None]: """The last real human ask and the last system prompt; either is None if absent. @@ -365,7 +384,7 @@ def _extract_current_ask_and_system_prompt( the caller routes to its default model. That is the correct answer rather than a gap to fill: filling it would hand tier selection to harness-injected text. """ - current_ask: Final = next(_iter_human_asks_newest_first(messages, markers), None) + current_ask: Final = next(_iter_human_asks_newest_first(messages, marker_pairs), None) system_prompt: Final = next( ( text @@ -385,7 +404,7 @@ def _truncate(text: str, limit: int) -> str: def _iter_context_turns_newest_first( messages: Sequence[Mapping[str, object]], include_assistant: bool, - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> Iterator[tuple[str, str]]: """Yield (role, text) for turns eligible as classifier context, newest first. @@ -401,7 +420,7 @@ def _iter_context_turns_newest_first( for msg in reversed(messages) if isinstance(role := msg.get("role"), str) and role in roles - and (text := _human_text(msg.get("content"), *markers)) + and (text := _human_text(msg.get("content"), marker_pairs)) ) @@ -411,7 +430,7 @@ def _extract_prior_turns( window_size: int, per_turn_chars: int, include_assistant: bool, - markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE), + marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS, ) -> tuple[tuple[str, str], ...]: """Up to window_size turns other than current_ask, oldest first, as (role, text). @@ -431,7 +450,7 @@ def _extract_prior_turns( prior: Final = islice( ( turn - for turn in _iter_context_turns_newest_first(messages, include_assistant, markers) + for turn in _iter_context_turns_newest_first(messages, include_assistant, marker_pairs) if turn[1] != current_ask ), window_size, @@ -556,7 +575,11 @@ def __init__( if self.config.escalation_keywords is not None else DEFAULT_ESCALATION_KEYWORDS ) - self._reminder_markers: tuple[str, str] = self.config.reminder_markers or (_REMINDER_OPEN, _REMINDER_CLOSE) + self._reminder_markers: tuple[tuple[str, str], ...] = ( + tuple((pair.open, pair.close) for pair in self.config.reminder_markers) + if self.config.reminder_markers + else _DEFAULT_REMINDER_MARKERS + ) # Lazily built on first semantic request and cached for reuse (route # embeddings are static, only the prompt is embedded per request). The lock @@ -993,7 +1016,7 @@ async def _classify_with_llm( window_size=self.config.classifier_context_window_size, per_turn_chars=self.config.classifier_context_per_turn_chars, include_assistant=include_assistant, - markers=self._reminder_markers, + marker_pairs=self._reminder_markers, ) if context_enabled else () diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py index f9d3bd9ae67f..69609a973b02 100644 --- a/litellm/router_strategy/complexity_router/config.py +++ b/litellm/router_strategy/complexity_router/config.py @@ -59,6 +59,30 @@ def _normalize_keywords(self) -> "KeywordTierRule": return self +class ReminderMarkerPair(BaseModel): + """One open/close delimiter pair a harness wraps injected context in. + + Normalizing here rather than at the scan is what makes matching case-insensitive: markers reach + the scan already lowered, so it lowercases only the haystack and never the needles. Stripping + keeps YAML indentation whitespace from becoming part of the delimiter. + """ + + open: str = Field(description="Opening delimiter, e.g. ''") + close: str = Field(description="Closing delimiter, e.g. ''") + + @model_validator(mode="after") + def _normalize(self) -> "ReminderMarkerPair": + open_marker: Final = self.open.strip().lower() + close_marker: Final = self.close.strip().lower() + if not open_marker or not close_marker: + raise ValueError("reminder_markers entries must not be blank") + if open_marker == close_marker: + raise ValueError("reminder_markers open and close must be different strings") + self.open = open_marker + self.close = close_marker + return self + + # ─── Default Keyword Lists ─── # Note: Keywords should be full words/phrases to avoid substring false positives. # The matching logic uses word boundary detection for single-word keywords. @@ -498,12 +522,15 @@ class ComplexityRouterConfig(BaseModel): description="RoutingPlugin instances that narrow the classified tier's candidate models before selection", ) - reminder_markers: tuple[str, str] | None = Field( + reminder_markers: tuple[ReminderMarkerPair, ...] | None = Field( default=None, + min_length=1, description=( - "Override the (open, close) marker pair used to recognize and strip harness-injected " - "reminder blocks before classification. Defaults to Claude Code's convention, " - "('', ''), when unset. Matching is case-insensitive." + "Override the delimiter pairs used to recognize and strip harness-injected reminder " + "blocks before classification. A harness that wraps injected context differently per " + "agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than " + "adds to, the built-in default of ('', ''), so a " + "harness that also emits that pair lists it too. Matching is case-insensitive." ), ) @@ -601,18 +628,6 @@ def _validate_plugins_adaptive_combo(self) -> "ComplexityRouterConfig": ) return self - @model_validator(mode="after") - def _normalize_reminder_markers(self) -> "ComplexityRouterConfig": - if self.reminder_markers is None: - return self - open_marker, close_marker = (marker.strip().lower() for marker in self.reminder_markers) - if not open_marker or not close_marker: - raise ValueError("reminder_markers entries must not be blank") - if open_marker == close_marker: - raise ValueError("reminder_markers open and close must be different strings") - self.reminder_markers = (open_marker, close_marker) - return self - def tier_label(self, tier: ComplexityTier) -> str: """Operator-facing display name for a tier, falling back to its canonical name.""" return self.tier_labels.get(tier, "").strip() or tier.value diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py index b3f1e9297413..8e9e32f58984 100644 --- a/tests/test_litellm/router_strategy/test_complexity_router.py +++ b/tests/test_litellm/router_strategy/test_complexity_router.py @@ -112,6 +112,52 @@ def test_init_with_config(self, mock_router_instance, basic_config): assert router.config.tiers["SIMPLE"] == "gpt-4o-mini" assert router.config.tiers["REASONING"] == "o1-preview" + def test_configured_marker_pairs_reach_the_ask_extraction(self, mock_router_instance, basic_config): + """Marker pairs configured in YAML must actually reach the code that strips them. + + The config field, the validator and the scan were each covered on their own, but nothing + exercised config.reminder_markers -> self._reminder_markers, so the router could have parsed + a valid config and still classified on unstripped text. Asserting through the extraction the + router feeds its classifier is what makes that wiring a regression rather than a silent gap. + """ + from litellm.router_strategy.complexity_router.complexity_router import ( + _extract_current_ask_and_system_prompt, + ) + + ask = "Derive the amortized complexity of a splay tree access" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config={ + **basic_config, + "reminder_markers": [ + {"open": "<<>>", "close": "<<>>"}, + {"open": "[[SUBAGENT_BEGIN]]", "close": "[[SUBAGENT_END]]"}, + ], + }, + ) + + assert router._reminder_markers == ( + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ) + messages = [ + {"role": "user", "content": ask}, + {"role": "assistant", "content": "Working on it."}, + {"role": "user", "content": "[[SUBAGENT_BEGIN]]Budget: 42 tokens remaining.[[SUBAGENT_END]]"}, + ] + assert _extract_current_ask_and_system_prompt(messages, router._reminder_markers)[0] == ask + + def test_unconfigured_marker_pairs_fall_back_to_the_builtin_default(self, mock_router_instance, basic_config): + """A config that never mentions reminder_markers keeps stripping .""" + router = ComplexityRouter( + model_name="test-router", + litellm_router_instance=mock_router_instance, + complexity_router_config=basic_config, + ) + + assert router._reminder_markers == (("", ""),) + def test_init_without_config(self, mock_router_instance): """Test initialization without configuration uses defaults.""" router = ComplexityRouter( @@ -2991,17 +3037,68 @@ def test_reminder_markers_unset_defaults_to_none(self): def test_reminder_markers_are_normalized(self): """Markers are stripped and lowercased, matching how the built-in constants are compared.""" config = ComplexityRouterConfig( - reminder_markers=(" <<>> ", "<<>>"), + reminder_markers=[{"open": " <<>> ", "close": "<<>>"}], + ) + assert config.reminder_markers is not None + assert (config.reminder_markers[0].open, config.reminder_markers[0].close) == ( + "<<>>", + "<<>>", ) - assert config.reminder_markers == ("<<>>", "<<>>") + + def test_reminder_markers_keep_every_configured_pair_in_order(self): + """Every pair a harness emits survives validation, not just the first.""" + config = ComplexityRouterConfig( + reminder_markers=[ + {"open": "<<>>", "close": "<<>>"}, + {"open": "[[SUBAGENT_BEGIN]]", "close": "[[SUBAGENT_END]]"}, + {"open": "%%CRON_BEGIN%%", "close": "%%CRON_END%%"}, + ], + ) + assert config.reminder_markers is not None + assert [(pair.open, pair.close) for pair in config.reminder_markers] == [ + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ("%%cron_begin%%", "%%cron_end%%"), + ] def test_reminder_markers_reject_blank_entry(self): with pytest.raises(ValidationError, match="must not be blank"): - ComplexityRouterConfig(reminder_markers=("", "<<>>")) + ComplexityRouterConfig(reminder_markers=[{"open": "", "close": "<<>>"}]) def test_reminder_markers_reject_identical_open_and_close(self): with pytest.raises(ValidationError, match="must be different"): - ComplexityRouterConfig(reminder_markers=("<<>>", "<<>>")) + ComplexityRouterConfig(reminder_markers=[{"open": "<<>>", "close": "<<>>"}]) + + def test_reminder_markers_reject_a_bad_pair_anywhere_in_the_list(self): + """Validation runs per pair, so a broken entry after a good one is still caught.""" + with pytest.raises(ValidationError, match="must be different"): + ComplexityRouterConfig( + reminder_markers=[ + {"open": "<<>>", "close": "<<>>"}, + {"open": "<<>>", "close": "<<>>"}, + ], + ) + + def test_reminder_markers_reject_empty_list(self): + """An explicitly empty list is ambiguous, so it fails loudly instead of silently defaulting. + + Left to fall through, an empty list resolves to the built-in pair, which + reads as "strip nothing" in the config and does the opposite. Matching on the length error + keeps this from passing for some unrelated reason if the field type changes. + """ + with pytest.raises(ValidationError, match="at least 1 item"): + ComplexityRouterConfig(reminder_markers=[]) + + def test_reminder_markers_reject_the_old_flat_pair_form(self): + """The pre-list shape is rejected loudly rather than silently routing on unstripped text. + + reminder_markers took a bare (open, close) string pair before it took a list of pairs. A + config still using that shape must fail validation at startup and at /model/new write time, + because the alternative -- accepting it and stripping nothing -- hands tier selection, and + therefore spend, to harness-injected text without any signal that it happened. + """ + with pytest.raises(ValidationError, match="valid dictionary or instance of ReminderMarkerPair"): + ComplexityRouterConfig(reminder_markers=("", "")) class _StubEncoder: @@ -4306,7 +4403,6 @@ async def test_reasoning_override_is_its_own_cause(self, complexity_router): # The score is still recorded, but the cause is what says it did not decide. assert decision["score"] < decision["tier_boundaries"]["complex_reasoning"] - @pytest.mark.asyncio async def test_an_unrenamed_router_writes_no_tier_label(self, complexity_router): """Renaming is opt-in, so a deployment that never renamed must gain no new key. @@ -4919,12 +5015,73 @@ def test_custom_markers_skip_a_reminder_only_follow_up_message(self): """ from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt - markers = ("<<>>", "<<>>") - follow_up_reminder = f"{markers[0]}Budget: 42 tokens remaining. Do not mention this.{markers[1]}" + pair = ("<<>>", "<<>>") + follow_up_reminder = f"{pair[0]}Budget: 42 tokens remaining. Do not mention this.{pair[1]}" messages = [_ASKED, _ANSWERED, {"role": "user", "content": follow_up_reminder}] assert _extract_current_ask_and_system_prompt(messages)[0] == follow_up_reminder - assert _extract_current_ask_and_system_prompt(messages, markers)[0] == _ASK + assert _extract_current_ask_and_system_prompt(messages, (pair,))[0] == _ASK + + def test_every_configured_marker_pair_is_stripped_not_just_the_first(self): + """One deployment serves a harness whose agent types each use a different envelope. + + Main agent, subagent and cron wrap injected context in different open/close pairs, and they + all route through the same auto-router. When only one pair could be configured, the other + agent types kept hitting the original bug: their reminder-only turn never stripped to empty, + won "newest human ask", and the harness blob got classified in place of the real question. + Each pair in turn must be skipped, so this fails if only the first configured pair is used. + """ + from litellm.router_strategy.complexity_router.complexity_router import _extract_current_ask_and_system_prompt + + pairs = ( + ("<<>>", "<<>>"), + ("[[subagent_begin]]", "[[subagent_end]]"), + ("%%cron_begin%%", "%%cron_end%%"), + ) + for open_marker, close_marker in pairs: + reminder_only_turn = f"{open_marker}Budget: 42 tokens remaining.{close_marker}" + messages = [_ASKED, _ANSWERED, {"role": "user", "content": reminder_only_turn}] + + assert _extract_current_ask_and_system_prompt(messages, pairs)[0] == _ASK, open_marker + + def test_a_block_nested_inside_another_pairs_block_does_not_leak(self): + """Nested blocks from two pairs must strip whole, not resume inside the outer block. + + Spans are collected per pair and can nest. Resuming the kept text at each block's own end + walks backwards into the enclosing block, so the outer block's remainder (and its dangling + close marker) survive into the classified ask. That is harness text choosing the tier, and + therefore the spend. Overlapping and disjoint spans strip correctly either way, so this + nested case is what pins the behavior. + """ + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + nested = "<<>>budget[[subagent_begin]]inner[[subagent_end]]do not mention<<>>" + + assert _strip_reminder_blocks(f"{nested} what is a splay tree?", pairs) == "what is a splay tree?" + + def test_overlapping_blocks_from_two_pairs_strip_whole(self): + """Interleaved (not nested) blocks still strip everything they jointly cover.""" + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + overlapping = "<<>>a[[subagent_begin]]b<<>>c[[subagent_end]]" + + assert _strip_reminder_blocks(f"{overlapping} what is a splay tree?", pairs) == "what is a splay tree?" + + def test_an_unclosed_marker_in_one_pair_does_not_suppress_another_pairs_blocks(self): + """Each pair scans independently, so one pair's dangling opener is not a global stop. + + An unclosed tag ends that pair's scan by design and is left intact as prose. It must not + also swallow a different pair's complete block, which would put harness text back in front + of the classifier. + """ + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + text = "<<>> why is [[subagent_begin]]noise[[subagent_end]] my tag stripped?" + + assert _strip_reminder_blocks(text, pairs) == "<<>> why is my tag stripped?" @pytest.mark.parametrize( "messages,current_ask,window,per_turn_chars,include_assistant,expected", @@ -5084,6 +5241,28 @@ def test_prior_turn_window(self, messages, current_ask, window, per_turn_chars, assert _extract_prior_turns(messages, current_ask, window, per_turn_chars, include_assistant) == expected + def test_prior_turn_context_strips_every_configured_pair(self): + """The classifier's context window is stripped with the same pairs as the ask. + + Prior turns are quoted verbatim into the LLM classifier payload, so a pair that is honored + when picking the ask but ignored when building context puts the harness blob back in front + of the classifier through the other door. This covers the _extract_prior_turns call the ask + extraction tests never reach. + """ + from litellm.router_strategy.complexity_router.complexity_router import _extract_prior_turns + + pairs = (("<<>>", "<<>>"), ("[[subagent_begin]]", "[[subagent_end]]")) + messages = [ + {"role": "user", "content": "[[subagent_begin]]budget blob[[subagent_end]]what about b-trees?"}, + {"role": "user", "content": "<<>>other blob<<>>and heaps?"}, + {"role": "user", "content": "current ask"}, + ] + + assert _extract_prior_turns(messages, "current ask", 5, 200, False, pairs) == ( + ("user", "what about b-trees?"), + ("user", "and heaps?"), + ) + def test_reminder_scan_is_linear_on_adversarial_input(self): """Unclosed reminder tags must not make stripping superlinear. @@ -5105,6 +5284,29 @@ def test_reminder_scan_is_linear_on_adversarial_input(self): assert elapsed < 1.0, f"stripping {len(adversarial)} chars took {elapsed:.2f}s; scan is not linear" assert result == adversarial + def test_reminder_scan_stays_linear_in_block_count_across_pairs(self): + """Many *complete* blocks across several pairs must not go quadratic either. + + Collapsing nested and overlapping spans is required for correctness once more than one pair + is configured, and the obvious way to write it -- folding merged spans into a growing tuple + -- is quadratic in block count. Unlike the unclosed-tag case above, these blocks all close, + so they actually produce spans. This input is a few hundred KB, which any keyholder can send + pre-routing, and it fails loudly if the collapse is ever rewritten as a fold. + """ + import time + + from litellm.router_strategy.complexity_router.complexity_router import _strip_reminder_blocks + + pairs = (("", ""), ("", "")) + adversarial = "xy" * 25_000 + + start = time.perf_counter() + result = _strip_reminder_blocks(f"{adversarial} what is a splay tree?", pairs) + elapsed = time.perf_counter() - start + + assert elapsed < 1.0, f"stripping {50_000} blocks took {elapsed:.2f}s; collapse is not linear" + assert result == "what is a splay tree?" + @pytest.mark.asyncio async def test_llm_classifier_includes_prior_turns_context(self, llm_complexity_router, mock_router_instance): """Test that the LLM classifier receives prior-turn context in the user message.""" @@ -5761,7 +5963,9 @@ def test_unset_system_prompt_defaults_to_none(self): @pytest.mark.asyncio async def test_custom_prompt_is_sent_verbatim_as_the_system_role(self, mock_router_instance, llm_classifier_config): - custom = "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + custom = ( + "Classify the data sensitivity: SIMPLE=public, MEDIUM=internal, COMPLEX=confidential, REASONING=regulated." + ) router = ComplexityRouter( model_name="test-complexity-router", litellm_router_instance=mock_router_instance, diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 65434407f74e..959c99218962 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -31560,6 +31560,26 @@ export interface components { /** Review Notes */ review_notes?: string | null; }; + /** + * ReminderMarkerPair + * @description One open/close delimiter pair a harness wraps injected context in. + * + * Normalizing here rather than at the scan is what makes matching case-insensitive: markers reach + * the scan already lowered, so it lowercases only the haystack and never the needles. Stripping + * keeps YAML indentation whitespace from becoming part of the delimiter. + */ + ReminderMarkerPair: { + /** + * Close + * @description Closing delimiter, e.g. '' + */ + close: string; + /** + * Open + * @description Opening delimiter, e.g. '' + */ + open: string; + }; /** * RequestComplexityRouterConfig * @description The part of a complexity-router config a request can carry. @@ -31672,12 +31692,9 @@ export interface components { reasoning_keywords?: string[] | null; /** * Reminder Markers - * @description Override the (open, close) marker pair used to recognize and strip harness-injected reminder blocks before classification. Defaults to Claude Code's convention, ('', ''), when unset. Matching is case-insensitive. + * @description Override the delimiter pairs used to recognize and strip harness-injected reminder blocks before classification. A harness that wraps injected context differently per agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than adds to, the built-in default of ('', ''), so a harness that also emits that pair lists it too. Matching is case-insensitive. */ - reminder_markers?: [ - string, - string - ] | null; + reminder_markers?: components["schemas"]["ReminderMarkerPair"][] | null; /** * Return Raw Model Name * @description Return the resolved raw model name in the response model field instead of the client-requested complexity-router alias