feat(spend): tell a mid-conversation switch from a conversation's first turn

litellm/constants.py

@@ -1655,3 +1655,10 @@
     "want to verify your reasoning, or face a complex decision. "
     "Describe your question or challenge clearly in the 'question' field."
 )
+
+########################### AUTO-ROUTER SAVINGS CONSTANTS ###########################
+# How long the model served to a session is remembered, so a later turn can tell a
+# model switch from a conversation that just started. Matches the session-affinity
+# pin's default lifetime; a conversation quieter than this is treated as new, which
+# falls back to the conservative savings rule rather than misattributing a switch.
+AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS: int = 3600

litellm/proxy/_types.py

@@ -3322,6 +3322,8 @@ class SpendLogsMetadata(TypedDict):
     cost_breakdown: Optional[CostBreakdown]  # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
     compression_savings: CompressionSavingsMetadata | None
     auto_router_savings_baseline_model: str | None  # counterfactual model for the auto-router savings driver
+    auto_router_previous_model: str | None  # model this session was served last; tells a switch from a first turn
+    auto_router_session_tracked: bool | None  # whether the request named a session at all
 
 
 class SpendLogsPayload(TypedDict):

litellm/proxy/db/db_spend_update_writer.py

@@ -1869,6 +1869,8 @@ async def _common_add_spend_log_transaction_to_daily_transaction(
                 compression_saved_tokens=compression_saved_tokens,
                 cache_read_input_tokens=cache_read_input_tokens,
                 baseline_model=_metadata.get("auto_router_savings_baseline_model"),
+                previous_model=_metadata.get("auto_router_previous_model"),
+                session_tracked=bool(_metadata.get("auto_router_session_tracked")),
                 usage_object=usage_obj,
             )
 

litellm/proxy/litellm_pre_call_utils.py

@@ -204,6 +204,8 @@ def parse_cache_control(cache_control):
     "policy_sources",
     "routing_decision",
     "auto_router_savings_baseline_model",
+    "auto_router_previous_model",
+    "auto_router_session_tracked",
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     "standard_logging_object",
     "proxy_server_request",

litellm/proxy/spend_tracking/savings.py

@@ -103,16 +103,36 @@ def _cache_token_split(usage: Usage) -> tuple[int, int]:
 )
 
 
-def _baseline_usage(usage: Usage) -> Usage:
+def _is_mid_conversation_switch(
+    previous_model: _ModelIdentity | None, selected: _ModelIdentity, session_tracked: bool
+) -> bool:
+    """Whether some other model was already warm for this conversation.
+
+    Three states, not two. An untracked request named no session at all, so nothing
+    can say why its cache is cold and it stays on the conservative rule, which
+    under-claims rather than inflates. A tracked request with no previous model is a
+    genuine first turn, and one whose previous model is the selected one stayed put;
+    neither is a switch, and both would have paid the same write on a single-model
+    deployment. Only a tracked request served something else before is.
+    """
+    if not session_tracked:
+        return True
+    return previous_model is not None and previous_model != selected
+
+
+def _baseline_usage(usage: Usage, is_switch: bool) -> Usage:
     """The same request as a single-model baseline would have met it.
 
-    Staying on one model, the prompt is written to cache once and read from thereafter,
-    so whatever this request paid to write would already have been cached on the
-    baseline. That holds whether or not this request also read anything: a switch to a
-    cold model reads nothing precisely because its cache is empty, which is the case the
-    penalty exists for. Gating on a read instead would charge the baseline a write it
-    would never repeat, and a cold switch would then report a larger saving than the
-    same traffic with caching turned off.
+    On a switch, the model that was already serving this conversation had the prompt
+    cached, so whatever this request paid to write would have been a read on the
+    baseline. The write is the switch's own cost and has to count against the saving,
+    which is why the cache tokens move into the read bucket here.
+
+    On a first turn nothing was cached anywhere, so the baseline would have paid the
+    same write; leaving the usage alone lets both arms carry it and the saving comes
+    out as the rate difference it really is. Charging the write to both cases, which
+    is what having no discriminator forces, understates a genuine first turn to a
+    few percent of its value and can render it as a loss.
 
     Only the cache buckets move. Every other field the request was priced on travels
     through untouched, audio and image and video counts among them, because the baseline
@@ -122,7 +142,7 @@ def _baseline_usage(usage: Usage) -> Usage:
     """
     cache_read, cache_creation = _cache_token_split(usage)
     details = usage.prompt_tokens_details
-    if details is None or cache_creation <= 0:
+    if details is None or cache_creation <= 0 or not is_switch:
         return usage
     return Usage(
         prompt_tokens=usage.prompt_tokens,
@@ -149,6 +169,8 @@ def compute_autorouter_savings(
     selected_model: str | None,
     selected_provider: str | None,
     usage: Usage,
+    previous_model: str | None = None,
+    session_tracked: bool = False,
 ) -> float:
     """Net dollars the router saved, or cost, by serving this request on ``selected_model``.
 
@@ -157,6 +179,11 @@ def compute_autorouter_savings(
     incurred; when that charge outweighs the cheaper rates, routing lost money and the
     dashboard has to be able to say so. Zero when both sides resolve to the same
     deployment, or when either cannot be resolved or priced.
+
+    ``previous_model`` is what this conversation was served last and ``session_tracked``
+    whether the request named a session at all. Together they separate a switch from a
+    first turn; without a session there is no discriminator, so every cold cache is
+    charged as a switch, which understates rather than inflates.
     """
     # No provider argument for the baseline on purpose: it arrives from the routing
     # metadata as a single self-describing string, already qualified by the auto-router,
@@ -165,7 +192,11 @@ def compute_autorouter_savings(
     selected = _resolve_model(selected_model, selected_provider)
     if baseline is None or selected is None or baseline == selected:
         return 0.0
-    baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage))
+    # Resolved, not string-compared: the previous model is recorded as the router's own
+    # model-group name while the selected one arrives normalized from the spend log, so
+    # raw equality reads `anthropic/claude-opus-5` as a switch away from `claude-opus-5`.
+    is_switch = _is_mid_conversation_switch(_resolve_model(previous_model, None), selected, session_tracked)
+    baseline_cost = _cost_of_usage(baseline, _baseline_usage(usage, is_switch=is_switch))
     selected_cost = _cost_of_usage(selected, usage)
     if baseline_cost is None or selected_cost is None:
         return 0.0
@@ -193,6 +224,8 @@ def compute_savings_spend(
     cache_read_input_tokens: int,
     baseline_model: str | None = None,
     usage_object: dict | None = None,
+    previous_model: str | None = None,
+    session_tracked: bool = False,
 ) -> SavingsSpend:
     """
     Dollar savings for one request, split by optimization driver.
@@ -201,7 +234,8 @@ def compute_savings_spend(
     input rate. Prompt-caching savings price the cache-read tokens at the
     difference between the input rate and the discounted cache-read rate.
     Auto-router savings compare the served ``model`` against the counterfactual
-    ``baseline_model`` and are zero unless the two differ.
+    ``baseline_model`` and are zero unless the two differ; ``previous_model``
+    tells a mid-conversation switch from a conversation's first turn.
     """
     input_cost, cache_read_cost = _input_and_cache_read_cost(model, custom_llm_provider)
     compression = max(compression_saved_tokens, 0) * input_cost
@@ -215,6 +249,8 @@ def compute_savings_spend(
         baseline_model=baseline_model,
         selected_model=model,
         selected_provider=custom_llm_provider,
+        previous_model=previous_model,
+        session_tracked=session_tracked,
         usage=usage,
     )
     return SavingsSpend(compression=compression, prompt_caching=prompt_caching, autorouter=autorouter)

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -118,6 +118,8 @@ def _get_spend_logs_metadata(
             cost_breakdown=None,
             compression_savings=None,
             auto_router_savings_baseline_model=None,
+            auto_router_previous_model=None,
+            auto_router_session_tracked=None,
             litellm_call_id=litellm_call_id,
         )
     verbose_proxy_logger.debug(

litellm/router.py

@@ -11224,8 +11224,21 @@ def _record_routing_decision(
             )
         if baseline_model is not None:
             recorded["auto_router_savings_baseline_model"] = baseline_model
-
-        cleared = {"routing_decision", "auto_router_savings_baseline_model"} - recorded.keys()
+        previous_model = pre_routing_hook_response.previous_model if pre_routing_hook_response else None
+        if previous_model is not None:
+            recorded["auto_router_previous_model"] = previous_model
+        # Recorded even when there is no previous model: a tracked session with none is a
+        # genuine first turn, which is priced the opposite way to a request that named no
+        # session at all. Collapsing the two into a missing key loses that distinction.
+        if pre_routing_hook_response is not None and pre_routing_hook_response.session_tracked:
+            recorded["auto_router_session_tracked"] = True
+
+        cleared = {
+            "routing_decision",
+            "auto_router_savings_baseline_model",
+            "auto_router_previous_model",
+            "auto_router_session_tracked",
+        } - recorded.keys()
         for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
             if isinstance(bucket, dict):
                 for key in cleared:

litellm/router_strategy/complexity_router/complexity_router.py

@@ -25,7 +25,11 @@
 from pydantic import BaseModel
 
 from litellm._logging import verbose_router_logger
-from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, RETURN_RAW_MODEL_NAME_METADATA_KEY
+from litellm.constants import (
+    AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS,
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    RETURN_RAW_MODEL_NAME_METADATA_KEY,
+)
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.llms.base_llm.base_utils import type_to_response_format_param
 from litellm.types.utils import (
@@ -1293,35 +1297,26 @@ def _extract_user_message_and_system_prompt(
         """
         return _extract_current_ask_and_system_prompt(messages)
 
-    @staticmethod
-    def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
-        """Metadata may land on `metadata` or `litellm_metadata` depending on the
-        endpoint, mirroring DeploymentAffinityCheck's precedence."""
-        return [
-            metadata
-            for metadata_key in ("litellm_metadata", "metadata")
-            if isinstance(metadata := request_kwargs.get(metadata_key), dict)
-        ]
-
     @staticmethod
     def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
-        """Resolve a client-supplied session_id."""
-        for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
-            session_id = metadata.get("session_id")
-            if session_id is not None:
-                return str(session_id)
-        return None
+        """Resolve the session this request belongs to.
+
+        Wider than a client-supplied `metadata.session_id`: the proxy also writes the
+        session id it derives from an `x-*-session-id` header or from Anthropic's
+        `metadata.user_id` into the same field before routing.
+        """
+        from litellm.router_utils.session_identity import session_id_from_request
+
+        return session_id_from_request(request_kwargs)
 
     @staticmethod
     def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
         """Resolve the proxy-derived API key hash, the same trust boundary
         DeploymentAffinityCheck uses for its own key-based affinity (not the
         client-supplied OpenAI `user` param, which isn't authenticated)."""
-        for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
-            user_key = metadata.get("user_api_key_hash")
-            if user_key is not None:
-                return str(user_key)
-        return None
+        from litellm.router_utils.session_identity import user_api_key_hash_from_request
+
+        return user_api_key_hash_from_request(request_kwargs)
 
     def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str:
         # Namespace by the caller's API key hash so two different callers reusing the
@@ -1331,6 +1326,54 @@ def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict)
         caller_scope = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped"
         return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}"
 
+    def _previous_model_cache_key(self, request_kwargs: dict) -> str | None:
+        """Where this session's last-served model is remembered, or ``None`` if it names no session.
+
+        Separate from the session-affinity key on purpose: this is recorded on every
+        request whether or not affinity is enabled, and the two must not read each
+        other's values.
+        """
+        from litellm.router_utils.session_identity import session_scope
+
+        return session_scope(
+            request_kwargs, namespace=self.model_name, discriminator="complexity_router_previous_model"
+        )
+
+    async def _previous_model_for_session(self, cache_key: str | None) -> str | None:
+        """The model this conversation was served last, or ``None`` if it is new.
+
+        Savings are priced against a counterfactual single-model deployment, and a cold
+        cache alone cannot say whether this request is cold because the router switched
+        models or because the conversation just started. Those two want opposite
+        arithmetic, so what the session was served last is the discriminator: different
+        from the model picked now means the cache write is the switch's own cost, absent
+        or equal means a single-model deployment would have paid it too.
+
+        Observation only. Nothing read here pins a deployment or narrows the candidate
+        pool; `session_affinity` is a separate feature and is unaffected.
+        """
+        if cache_key is None:
+            return None
+        try:
+            previous_model = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
+        except Exception as e:  # noqa: BLE001  # a dashboard metric must not fail a live request
+            verbose_router_logger.debug("complexity router: could not read the session's previous model (%s)", e)
+            return None
+        return previous_model if isinstance(previous_model, str) else None
+
+    async def _remember_model_for_session(self, cache_key: str | None, routed_model: str) -> None:
+        """Record what this session was served, for the next turn to compare against."""
+        if cache_key is None:
+            return
+        try:
+            await self.litellm_router_instance.cache.async_set_cache(
+                key=cache_key,
+                value=routed_model,
+                ttl=AUTOROUTER_PREVIOUS_MODEL_TTL_SECONDS,
+            )
+        except Exception as e:  # noqa: BLE001  # a dashboard metric must not fail a live request
+            verbose_router_logger.debug("complexity router: could not record the session's model (%s)", e)
+
     async def async_pre_routing_hook(
         self,
         model: str,
@@ -1359,6 +1402,9 @@ async def async_pre_routing_hook(
             if isinstance(metadata, dict):
                 metadata[RETURN_RAW_MODEL_NAME_METADATA_KEY] = True
 
+        previous_model_key = self._previous_model_cache_key(request_kwargs)
+        previous_model = await self._previous_model_for_session(previous_model_key)
+
         use_session_affinity = self.config.session_affinity and not self.config.plugins
         session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
         cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
@@ -1397,10 +1443,13 @@ async def async_pre_routing_hook(
                         f"ComplexityRouter: routing decision cause={cause}, routed_model={routed_model}"
                     )
                     has_original_messages = messages is not None and len(messages) > 0
+                    await self._remember_model_for_session(previous_model_key, routed_model)
                     return PreRoutingHookResponse(
                         model=routed_model,
                         messages=messages if has_original_messages else None,
                         savings_baseline_model=self.savings_baseline_model,
+                        previous_model=previous_model,
+                        session_tracked=previous_model_key is not None,
                         routing_decision=self._build_routing_decision(
                             routed_model=routed_model,
                             cause=cause,
@@ -1415,7 +1464,11 @@ async def async_pre_routing_hook(
             messages=messages,
             input=input,
             specific_deployment=specific_deployment,
+            previous_model=previous_model,
+            session_tracked=previous_model_key is not None,
         )
+        if response is not None:
+            await self._remember_model_for_session(previous_model_key, response.model)
         if cache_key is not None and response is not None:
             await self.litellm_router_instance.cache.async_set_cache(
                 key=cache_key,
@@ -1431,6 +1484,8 @@ async def _classify_and_route(
         messages: list[dict[str, Any]] | None = None,
         input: Union[str, list] | None = None,
         specific_deployment: bool | None = False,
+        previous_model: str | None = None,
+        session_tracked: bool = False,
     ) -> PreRoutingHookResponse | None:
         """
         Classifies the request by complexity and returns the appropriate model.
@@ -1478,6 +1533,8 @@ async def _classify_and_route(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
                 savings_baseline_model=self.savings_baseline_model,
+                previous_model=previous_model,
+                session_tracked=session_tracked,
                 routing_decision=self._build_routing_decision(routed_model=routed_model, cause="default_fallback"),
             )
 
@@ -1500,6 +1557,8 @@ async def _classify_and_route(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
                 savings_baseline_model=self.savings_baseline_model,
+                previous_model=previous_model,
+                session_tracked=session_tracked,
                 routing_decision=self._build_routing_decision(
                     routed_model=routed_model,
                     cause=keyword_cause,
@@ -1548,6 +1607,8 @@ async def _classify_and_route(
             model=routed_model,
             messages=messages if has_original_messages else None,
             savings_baseline_model=self.savings_baseline_model,
+            previous_model=previous_model,
+            session_tracked=session_tracked,
             routing_decision=self._build_routing_decision(
                 routed_model=routed_model,
                 cause=outcome.cause,