Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions litellm/router_strategy/complexity_router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,27 @@ If 2+ reasoning markers are detected in the user message, the request is automat

Reasoning markers in the system prompt do **not** trigger the reasoning override. This prevents system prompts like "Think step by step before answering" from forcing all requests to the reasoning tier.

### Harness Reminder Blocks

Agent harnesses inject their own context into the conversation as ordinary message text. That text is plumbing, not something a human asked for, so the router strips complete reminder blocks before classifying and picking a tier. A turn that is nothing but a reminder block strips to empty and is skipped, and the router falls back to the last real ask instead

By default a block is anything between `<system-reminder>` and `</system-reminder>`. `reminder_markers` replaces that with your harness's own delimiters. Many harnesses use a different envelope per agent type, so list every pair you emit:

```yaml
model_list:
- model_name: smart-router
litellm_params:
model: auto_router/complexity_router
complexity_router_config:
reminder_markers:
- open: "<<<BEGIN_CONTEXT>>>"
close: "<<<END_CONTEXT>>>"
- open: "[[SUBAGENT_CONTEXT_BEGIN]]"
close: "[[SUBAGENT_CONTEXT_END]]"
```

Setting `reminder_markers` replaces the built-in `<system-reminder>` pair rather than adding to it, so list that pair too if your harness also emits it. Matching is case-insensitive. Blocks that nest or overlap across pairs are stripped whole. An unclosed delimiter is not a block and is left in place, which keeps prose that merely mentions a delimiter from being eaten

Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
### Code Detection

Technical code keywords are detected case-insensitively and include:
Expand Down
2 changes: 2 additions & 0 deletions litellm/router_strategy/complexity_router/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
DEFAULT_COMPLEXITY_CONFIG,
ComplexityRouterConfig,
ComplexityTier,
ReminderMarkerPair,
)

__all__ = [
Expand All @@ -24,5 +25,6 @@
"ComplexityRouter",
"ComplexityRouterConfig",
"ComplexityTier",
"ReminderMarkerPair",
"classification_system_prompt",
]
71 changes: 47 additions & 24 deletions litellm/router_strategy/complexity_router/complexity_router.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
import random
import re
from collections.abc import Iterator, Mapping, Sequence
from itertools import islice
from itertools import accumulate, islice
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast

Expand Down Expand Up @@ -233,6 +233,7 @@ def _effective_turn_off_message_logging(request_kwargs: Mapping[str, Any] | None

_REMINDER_OPEN: Final = "<system-reminder>"
_REMINDER_CLOSE: Final = "</system-reminder>"
_DEFAULT_REMINDER_MARKERS: Final = ((_REMINDER_OPEN, _REMINDER_CLOSE),)

_TRUNCATION_MARKER: Final = "..."

Expand All @@ -253,10 +254,8 @@ def _message_text(content: object) -> str:
return content if isinstance(content, str) else ""


def _reminder_block_spans(
lowered: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE
) -> Iterator[tuple[int, int]]:
"""Span of each complete reminder block, left to right.
def _reminder_block_spans(lowered: str, open_marker: str, close_marker: str) -> Iterator[tuple[int, int]]:
"""Span of each complete reminder block for one marker pair, left to right.

Literal `str.find`, not a regex: the delimiters are fixed strings, and `<system-reminder>.*?`
retried its lazy quantifier from every opening tag, so repeated unclosed tags were quadratic
Expand All @@ -272,17 +271,36 @@ def _reminder_block_spans(
yield start, cursor


def _strip_reminder_blocks(text: str, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str:
"""Remove every complete reminder block from text, keeping everything written around them."""
spans: Final = tuple(_reminder_block_spans(text.lower(), open_marker, close_marker))
def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str:
"""Remove every complete reminder block from text, keeping everything written around them.

Blocks from different pairs can nest or overlap, which the gap construction below would
otherwise mishandle: an inner block's end would resume the kept text partway through the outer
block, leaking the rest of that block into the classified ask. Running the block ends through a
maximum resumes each gap past the furthest block seen so far, which collapses nested and
overlapping spans without a separate merge pass. A single pair's ends already increase, so the
maximum is the identity there and the default path is byte-identical to a plain scan.

Deliberately linear in both the text and the block count. This runs pre-routing on input any
keyholder controls, and both a regex scan and a fold that rebuilds a growing tuple of merged
spans go quadratic on inputs that are cheap to send.
"""
lowered: Final = text.lower()
spans: Final = tuple(
sorted(
span
for open_marker, close_marker in marker_pairs
for span in _reminder_block_spans(lowered, open_marker, close_marker)
)
)
if not spans:
return text.strip()
keep_from: Final = (0, *(end for _, end in spans))
keep_from: Final = (0, *accumulate((end for _, end in spans), max))
keep_to: Final = (*(start for start, _ in spans), len(text))
return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip()))


def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker: str = _REMINDER_CLOSE) -> str:
def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str:
"""Message content as the text a human wrote, with complete reminder blocks removed.

Harnesses inject reminders as ordinary text alongside the live ask, so the block is stripped and
Expand All @@ -291,18 +309,18 @@ def _human_text(content: object, open_marker: str = _REMINDER_OPEN, close_marker
one, and this same string drives escalation keywords and keyword_tier_rules, which choose the
model and therefore the spend. An unclosed tag is not a block and is left intact.
"""
return _strip_reminder_blocks(_message_text(content), open_marker, close_marker)
return _strip_reminder_blocks(_message_text(content), marker_pairs)


def _iter_human_asks_newest_first(
messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE)
messages: Sequence[Mapping[str, object]],
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
) -> Iterator[str]:
"""Yield user-turn texts that carry a real human ask, newest first, with harness noise removed."""
open_marker, close_marker = markers
return (
text
for msg in reversed(messages)
if msg.get("role") == "user" and (text := _human_text(msg.get("content"), open_marker, close_marker))
if msg.get("role") == "user" and (text := _human_text(msg.get("content"), marker_pairs))
)


Expand Down Expand Up @@ -341,7 +359,8 @@ def _conversation_is_continuing(messages: Sequence[Mapping[str, object]] | None)


def _newest_turn_ask(
messages: Sequence[Mapping[str, object]], markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE)
messages: Sequence[Mapping[str, object]],
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
) -> str | None:
"""The human ask on the newest user turn, or None when that turn carries only plumbing.

Expand All @@ -352,20 +371,20 @@ def _newest_turn_ask(
newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None)
if newest_user_turn is None:
return None
return _human_text(newest_user_turn.get("content"), *markers) or None
return _human_text(newest_user_turn.get("content"), marker_pairs) or None


def _extract_current_ask_and_system_prompt(
messages: Sequence[Mapping[str, object]],
markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE),
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
) -> tuple[str | None, str | None]:
"""The last real human ask and the last system prompt; either is None if absent.

A conversation whose every user turn is only plumbing has no ask, so `current_ask` is None and
the caller routes to its default model. That is the correct answer rather than a gap to fill:
filling it would hand tier selection to harness-injected text.
"""
current_ask: Final = next(_iter_human_asks_newest_first(messages, markers), None)
current_ask: Final = next(_iter_human_asks_newest_first(messages, marker_pairs), None)
system_prompt: Final = next(
(
text
Expand All @@ -385,7 +404,7 @@ def _truncate(text: str, limit: int) -> str:
def _iter_context_turns_newest_first(
messages: Sequence[Mapping[str, object]],
include_assistant: bool,
markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE),
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
) -> Iterator[tuple[str, str]]:
"""Yield (role, text) for turns eligible as classifier context, newest first.

Expand All @@ -401,7 +420,7 @@ def _iter_context_turns_newest_first(
for msg in reversed(messages)
if isinstance(role := msg.get("role"), str)
and role in roles
and (text := _human_text(msg.get("content"), *markers))
and (text := _human_text(msg.get("content"), marker_pairs))
)


Expand All @@ -411,7 +430,7 @@ def _extract_prior_turns(
window_size: int,
per_turn_chars: int,
include_assistant: bool,
markers: tuple[str, str] = (_REMINDER_OPEN, _REMINDER_CLOSE),
marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS,
) -> tuple[tuple[str, str], ...]:
"""Up to window_size turns other than current_ask, oldest first, as (role, text).

Expand All @@ -431,7 +450,7 @@ def _extract_prior_turns(
prior: Final = islice(
(
turn
for turn in _iter_context_turns_newest_first(messages, include_assistant, markers)
for turn in _iter_context_turns_newest_first(messages, include_assistant, marker_pairs)
if turn[1] != current_ask
),
window_size,
Expand Down Expand Up @@ -556,7 +575,11 @@ def __init__(
if self.config.escalation_keywords is not None
else DEFAULT_ESCALATION_KEYWORDS
)
self._reminder_markers: tuple[str, str] = self.config.reminder_markers or (_REMINDER_OPEN, _REMINDER_CLOSE)
self._reminder_markers: tuple[tuple[str, str], ...] = (
tuple((pair.open, pair.close) for pair in self.config.reminder_markers)
if self.config.reminder_markers
else _DEFAULT_REMINDER_MARKERS
)

# Lazily built on first semantic request and cached for reuse (route
# embeddings are static, only the prompt is embedded per request). The lock
Expand Down Expand Up @@ -993,7 +1016,7 @@ async def _classify_with_llm(
window_size=self.config.classifier_context_window_size,
per_turn_chars=self.config.classifier_context_per_turn_chars,
include_assistant=include_assistant,
markers=self._reminder_markers,
marker_pairs=self._reminder_markers,
)
if context_enabled
else ()
Expand Down
47 changes: 31 additions & 16 deletions litellm/router_strategy/complexity_router/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,30 @@ def _normalize_keywords(self) -> "KeywordTierRule":
return self


class ReminderMarkerPair(BaseModel):
"""One open/close delimiter pair a harness wraps injected context in.

Normalizing here rather than at the scan is what makes matching case-insensitive: markers reach
the scan already lowered, so it lowercases only the haystack and never the needles. Stripping
keeps YAML indentation whitespace from becoming part of the delimiter.
"""

open: str = Field(description="Opening delimiter, e.g. '<system-reminder>'")
close: str = Field(description="Closing delimiter, e.g. '</system-reminder>'")

@model_validator(mode="after")
def _normalize(self) -> "ReminderMarkerPair":
open_marker: Final = self.open.strip().lower()
close_marker: Final = self.close.strip().lower()
if not open_marker or not close_marker:
raise ValueError("reminder_markers entries must not be blank")
if open_marker == close_marker:
raise ValueError("reminder_markers open and close must be different strings")
self.open = open_marker
self.close = close_marker
return self


# ─── Default Keyword Lists ───
# Note: Keywords should be full words/phrases to avoid substring false positives.
# The matching logic uses word boundary detection for single-word keywords.
Expand Down Expand Up @@ -498,12 +522,15 @@ class ComplexityRouterConfig(BaseModel):
description="RoutingPlugin instances that narrow the classified tier's candidate models before selection",
)

reminder_markers: tuple[str, str] | None = Field(
reminder_markers: tuple[ReminderMarkerPair, ...] | None = Field(
default=None,
min_length=1,
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
description=(
"Override the (open, close) marker pair used to recognize and strip harness-injected "
"reminder blocks before classification. Defaults to Claude Code's convention, "
"('<system-reminder>', '</system-reminder>'), when unset. Matching is case-insensitive."
"Override the delimiter pairs used to recognize and strip harness-injected reminder "
"blocks before classification. A harness that wraps injected context differently per "
"agent type (main, subagent, cron) lists every pair it emits. Replaces, rather than "
"adds to, the built-in default of ('<system-reminder>', '</system-reminder>'), so a "
"harness that also emits that pair lists it too. Matching is case-insensitive."
),
)

Expand Down Expand Up @@ -601,18 +628,6 @@ def _validate_plugins_adaptive_combo(self) -> "ComplexityRouterConfig":
)
return self

@model_validator(mode="after")
def _normalize_reminder_markers(self) -> "ComplexityRouterConfig":
if self.reminder_markers is None:
return self
open_marker, close_marker = (marker.strip().lower() for marker in self.reminder_markers)
if not open_marker or not close_marker:
raise ValueError("reminder_markers entries must not be blank")
if open_marker == close_marker:
raise ValueError("reminder_markers open and close must be different strings")
self.reminder_markers = (open_marker, close_marker)
return self

def tier_label(self, tier: ComplexityTier) -> str:
"""Operator-facing display name for a tier, falling back to its canonical name."""
return self.tier_labels.get(tier, "").strip() or tier.value
Expand Down
Loading
Loading