fix(caching): skip the cache past max_messages and keep tool_result text in semantic prompts

litellm-rust/crates/cache/src/semantic.rs

@@ -83,17 +83,16 @@ impl Embedder for PreparedEmbedding {
     }
 }
 
-/// `get_str_from_messages`: every message's text content followed by its search results.
+/// `get_semantic_cache_prompt_from_messages`: every message's text content, including the text of
+/// Messages API `tool_result` blocks, followed by its search results.
 pub fn str_from_messages(messages: &[Value]) -> String {
     let mut text = String::new();
     for message in messages.iter().filter_map(Value::as_object) {
         match message.get("content") {
             Some(Value::String(content)) => text.push_str(content),
-            Some(Value::Array(parts)) => {
-                for part in parts {
-                    if let Some(part_text) = part.get("text").and_then(Value::as_str) {
-                        text.push_str(part_text);
-                    }
+            Some(Value::Array(blocks)) => {
+                for block in blocks {
+                    push_block_text(&mut text, block);
                 }
             }
             _ => {}
@@ -103,6 +102,28 @@ pub fn str_from_messages(messages: &[Value]) -> String {
     text
 }
 
+fn push_block_text(text: &mut String, block: &Value) {
+    if block.get("type").and_then(Value::as_str) != Some("tool_result") {
+        push_text_field(text, block);
+        return;
+    }
+    match block.get("content") {
+        Some(Value::String(result)) => text.push_str(result),
+        Some(Value::Array(blocks)) => {
+            for inner in blocks {
+                push_text_field(text, inner);
+            }
+        }
+        _ => {}
+    }
+}
+
+fn push_text_field(text: &mut String, block: &Value) {
+    if let Some(block_text) = block.get("text").and_then(Value::as_str) {
+        text.push_str(block_text);
+    }
+}
+
 /// The messages prompt Qdrant embeds: `None` when the request carries no messages.
 pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option<String> {
     let messages = context.messages.as_ref()?.as_array()?;
@@ -163,6 +184,10 @@ fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
                 collect_input_text(content, parts);
                 return;
             }
+            if let Some(output) = map.get("output").filter(|output| output.is_array()) {
+                collect_input_text(output, parts);
+                return;
+            }
             for key in ["text", "output", "input_text", "output_text"] {
                 if let Some(Value::String(text)) = map.get(key)
                     && push_trimmed(text, parts)

litellm-rust/crates/cache/tests/semantic.rs

@@ -30,6 +30,31 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
     ]}]),
     "What is this?",
 )]
+#[case::tool_result_string(
+    json!([
+        {"role": "user", "content": "list the files"},
+        {"role": "assistant", "content": [
+            {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}},
+        ]},
+        {"role": "user", "content": [
+            {"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"},
+        ]},
+    ]),
+    "list the filescalc.py test_calc.py",
+)]
+#[case::tool_result_blocks(
+    json!([{"role": "user", "content": [
+        {"type": "tool_result", "tool_use_id": "toolu_1", "content": [
+            {"type": "text", "text": "x = 1"},
+            {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}},
+        ]},
+    ]}]),
+    "x = 1",
+)]
+#[case::tool_result_without_content(
+    json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}]),
+    "",
+)]
 #[case::missing_null_and_empty_content(
     json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]),
     "",
@@ -166,6 +191,15 @@ fn prompt_from_messages_reads_messages_only(
     ])),
     Some("model dump prompt\ndict prompt\ninline prompt"),
 )]
+#[case::function_call_output_blocks(
+    None,
+    Some(json!([
+        {"role": "user", "content": "update the config"},
+        {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\": \"a\"}"},
+        {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]},
+    ])),
+    Some("update the config\nwrote a"),
+)]
 #[case::object_content(
     None,
     Some(json!({"content": [{"text": "object content prompt"}]})),

litellm/caching/caching.py

@@ -92,6 +92,17 @@ class CacheMode(str, Enum):
 
 
 #### LiteLLM.Completion / Embedding Cache ####
+def _request_message_count(kwargs: Mapping[str, object]) -> int:
+    """Chat and Messages API `messages`, else Responses API `input` items; embedding `input` strings count as none"""
+    messages: Final = kwargs.get("messages")
+    if isinstance(messages, list):
+        return len(messages)
+    input_items: Final = kwargs.get("input")
+    if not isinstance(input_items, list):
+        return 0
+    return sum(1 for item in input_items if isinstance(item, (Mapping, BaseModel)))
+
+
 class Cache:
     _native_cache: "ResponseCacheRuntime | None" = None
 
@@ -142,6 +153,7 @@ def __init__(
         semantic_cache_embedding_max_input_tokens: int | None = None,
         semantic_cache_embedding_timeout: float | None = None,
         semantic_cache_scope: str = SemanticCacheScope.KEY.value,
+        max_messages: int | None = 4,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -171,6 +183,7 @@ def __init__(
             semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
             semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS.
             semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key".
+            max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -321,6 +334,7 @@ def __init__(
         self.ttl = ttl
         self.mode: CacheMode = mode or CacheMode.default_on
         self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value
+        self.max_messages: int | None = max_messages
 
         if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None:
             self.ttl = default_in_memory_ttl
@@ -1005,7 +1019,10 @@ def should_use_cache(self, **kwargs):
 
         If cache is default_on then this is True
         If cache is default_off then this is only true when user has opted in to use cache
+        Always False once the request carries more than `max_messages` messages
         """
+        if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages:
+            return False
         if self.mode == CacheMode.default_on:
             return True
 

litellm/caching/qdrant_semantic_cache.py

@@ -24,7 +24,7 @@
 )
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    get_str_from_messages,
+    get_semantic_cache_prompt_from_messages,
 )
 from litellm.types.utils import EmbeddingResponse
 
@@ -286,7 +286,7 @@ def set_cache(self, key, value, **kwargs):
 
         # get the prompt
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # create an embedding for prompt
         embedding_response: Final = cast(
@@ -325,7 +325,7 @@ def get_cache(self, key, **kwargs):
 
         # get the messages
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         # convert to embedding
         embedding_response: Final = cast(
@@ -400,7 +400,7 @@ async def async_set_cache(self, key, value, **kwargs):
 
         # get the prompt
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
         embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
 
         # get the embedding
@@ -435,7 +435,7 @@ async def async_get_cache(self, key, **kwargs):
 
         # get the messages
         messages: Final = kwargs["messages"]
-        prompt: Final = get_str_from_messages(messages)
+        prompt: Final = get_semantic_cache_prompt_from_messages(messages)
 
         embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
 

litellm/caching/redis_semantic_cache.py

@@ -21,7 +21,7 @@
 from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    get_str_from_messages,
+    get_semantic_cache_prompt_from_messages,
 )
 from litellm.types.utils import EmbeddingResponse
 
@@ -263,7 +263,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None:
         """
         messages: Final = kwargs.get("messages")
         if messages:
-            return get_str_from_messages(messages)
+            return get_semantic_cache_prompt_from_messages(messages)
 
         if "input" not in kwargs:
             return None
@@ -291,9 +291,9 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -
             return
 
         if isinstance(value, dict):
-            content = value.get("content")
-            if content is not None:
-                cls._collect_responses_input_text(content, prompt_parts)
+            nested: Final = cls._nested_response_input(value)
+            if nested is not None:
+                cls._collect_responses_input_text(nested, prompt_parts)
                 return
 
             for text_key in ("text", "output", "input_text", "output_text"):
@@ -318,6 +318,14 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -
                     prompt_parts.append(stripped_text)
                     return
 
+    @staticmethod
+    def _nested_response_input(value: dict[str, object]) -> object:
+        content: Final = value.get("content")
+        if content is not None:
+            return content
+        output: Final = value.get("output")
+        return output if isinstance(output, list) else None
+
     @staticmethod
     def _coerce_response_input_value(value: object) -> object:
         model_dump: Final = getattr(value, "model_dump", None)

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -192,6 +192,33 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str:
     return text
 
 
+def get_semantic_cache_prompt_from_messages(messages: Sequence[Mapping[str, object]]) -> str:
+    """
+    The text a semantic cache embeds for a request: `get_str_from_messages` plus the text inside
+    Messages API `tool_result` blocks, so a tool turn does not embed identically to the turn before it
+    """
+    return "".join(
+        _semantic_cache_content_text(message.get("content"))
+        + extract_search_results_text(message.get("search_results"))
+        for message in messages
+    )
+
+
+def _semantic_cache_content_text(content: object) -> str:
+    if isinstance(content, str):
+        return content
+    if not isinstance(content, list):
+        return ""
+    return "".join(_semantic_cache_block_text(block) for block in content if isinstance(block, Mapping))
+
+
+def _semantic_cache_block_text(block: Mapping[str, object]) -> str:
+    if block.get("type") == "tool_result":
+        return _semantic_cache_content_text(block.get("content"))
+    text: Final = block.get("text")
+    return text if isinstance(text, str) else ""
+
+
 def is_non_content_values_set(message: AllMessageValues) -> bool:
     ignore_keys: Final = ["content", "role", "name"]
     return any(message.get(key, None) is not None for key in message if key not in ignore_keys)