Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
f462fbe
fix(caching): keep tool calls and tool results in semantic cache prompts
kerry-berri Sep 30, 2026
8b08e07
refactor(caching): keep semantic tool prompt helpers within lint budgets
kerry-berri Sep 30, 2026
8bf4439
fix(caching): keep structured function_call_output text in semantic p…
kerry-berri Sep 30, 2026
5165860
refactor(caching): split Responses text-field collection to stay with…
kerry-berri Sep 30, 2026
19bc489
fix(caching): tag each tool result with the position of the call it a…
kerry-berri Sep 30, 2026
6b4b375
fix(caching): encode tool result position and output together so tool…
kerry-berri Sep 30, 2026
9379f65
test(caching): expect encoded tool result record in qdrant semantic p…
kerry-berri Sep 30, 2026
1275062
test(caching): cover tool result arrangements, SDK clients, concurren…
kerry-berri Sep 30, 2026
5aed3ab
feat(caching): embed every semantic cache prompt field except volatil…
kerry-berri Sep 30, 2026
39efb5d
test(code-quality): allow the bounded semantic cache prompt walkers i…
kerry-berri Sep 30, 2026
86c82b9
test(rust): expect structured JSON for unknown fields in redis and va…
kerry-berri Sep 30, 2026
345bd97
Revert "test(rust): expect structured JSON for unknown fields in redi…
kerry-berri Sep 30, 2026
f8aad2f
Revert "test(code-quality): allow the bounded semantic cache prompt w…
kerry-berri Sep 30, 2026
ccca38f
Revert "feat(caching): embed every semantic cache prompt field except…
kerry-berri Sep 30, 2026
9de92d7
Merge remote-tracking branch 'origin/main' into litellm_semantic_cach…
kerry-berri Oct 6, 2026
b9a7942
refactor(caching): rename get_str_from_messages_with_tools to get_sem…
kerry-berri Oct 6, 2026
b27722c
refactor(caching): split semantic cache prompt extraction by API format
kerry-berri Oct 6, 2026
7c1390c
fix(caching): drop TypeIs guard and register Responses prompt walker …
kerry-berri Oct 6, 2026
1c4a69b
fix(caching): pick the Responses text field without a Final inside a …
kerry-berri Oct 6, 2026
4ac923b
refactor(caching): walk semantic cache prompts as plain dicts, dumpin…
kerry-berri Oct 6, 2026
ffd3a85
refactor(caching): write the semantic cache prompt builders as plain …
kerry-berri Oct 6, 2026
394abff
fix(caching): skip the cache past max_messages and keep tool_result t…
kerry-berri Oct 6, 2026
050e9c0
test(caching): drop formatting-only churn from the redis semantic cac…
kerry-berri Oct 6, 2026
6231a68
ci(integration): drop the caching group wiring that main already carries
kerry-berri Oct 6, 2026
c8ae23c
refactor(caching): recurse into tool_result content in the semantic c…
kerry-berri Oct 7, 2026
0dc9c91
test(integration): check the max_messages cap on the shared exact-cac…
kerry-berri Oct 7, 2026
f6e9be9
fix(caching): read list-form function_call_output text in semantic ca…
kerry-berri Oct 7, 2026
0665296
refactor(caching): extract nested Responses input lookup to keep walk…
kerry-berri Oct 7, 2026
ba93ca5
Revert "refactor(caching): extract nested Responses input lookup to k…
kerry-berri Oct 7, 2026
580a970
style(caching): suppress C901 on the Responses input walker instead o…
kerry-berri Oct 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 54 additions & 16 deletions litellm-rust/crates/cache/src/semantic.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
//! The embedding and prompt contract every semantic backend shares.
//!
//! Python's semantic caches all read their prompt through `get_str_from_messages`, and
//! Python's semantic caches all read their prompt through `get_str_from_messages_with_tools`, and
//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API
//! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here.

use std::{future::Future, io};

use serde::Serialize;
use serde_json::{
Value,
Value, json,
ser::{CharEscape, Formatter, Serializer},
};

Expand Down Expand Up @@ -83,26 +83,57 @@ impl Embedder for PreparedEmbedding {
}
}

/// `get_str_from_messages`: every message's text content followed by its search results.
/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results,
/// then its OpenAI `tool_calls`, then its search results.
pub fn str_from_messages(messages: &[Value]) -> String {
let mut text = String::new();
for message in messages.iter().filter_map(Value::as_object) {
match message.get("content") {
Some(Value::String(content)) => text.push_str(content),
Some(Value::Array(parts)) => {
for part in parts {
if let Some(part_text) = part.get("text").and_then(Value::as_str) {
text.push_str(part_text);
}
}
push_content_text(&mut text, message.get("content"));
if let Some(Value::Array(tool_calls)) = message.get("tool_calls") {
for tool_call in tool_calls.iter().filter_map(Value::as_object) {
let function = tool_call.get("function");
text.push_str(&tool_call_json(
function.and_then(|function| function.get("name")),
function.and_then(|function| function.get("arguments")),
));
}
_ => {}
}
push_search_results_text(&mut text, message.get("search_results"));
}
text
}

/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content.
fn push_content_text(text: &mut String, content: Option<&Value>) {
match content {
Some(Value::String(content)) => text.push_str(content),
Some(Value::Array(blocks)) => {
for block in blocks.iter().filter_map(Value::as_object) {
match block.get("type").and_then(Value::as_str) {
Some("tool_use") => {
text.push_str(&tool_call_json(block.get("name"), block.get("input")));
}
Some("tool_result") => push_content_text(text, block.get("content")),
_ => {
if let Some(block_text) = block.get("text").and_then(Value::as_str) {
text.push_str(block_text);
}
}
}
}
}
_ => {}
}
}

/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes.
fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String {
compact_json(&json!({
"name": name.unwrap_or(&Value::Null),
"arguments": arguments.unwrap_or(&Value::Null),
}))
}

/// The messages prompt Qdrant embeds: `None` when the request carries no messages.
pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option<String> {
let messages = context.messages.as_ref()?.as_array()?;
Expand Down Expand Up @@ -159,15 +190,22 @@ fn collect_input_text(value: &Value, parts: &mut Vec<String>) {
}
}
Value::Object(map) => {
if map.get("type").and_then(Value::as_str) == Some("function_call") {
parts.push(tool_call_json(map.get("name"), map.get("arguments")));
return;
}
if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
collect_input_text(content, parts);
return;
}
for key in ["text", "output", "input_text", "output_text"] {
if let Some(Value::String(text)) = map.get(key)
&& push_trimmed(text, parts)
{
return;
match map.get(key) {
Some(nested @ Value::Array(_)) => {
collect_input_text(nested, parts);
return;
}
Some(Value::String(text)) if push_trimmed(text, parts) => return,
_ => {}
}
}
}
Expand Down
56 changes: 56 additions & 0 deletions litellm-rust/crates/cache/tests/semantic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,45 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]),
"false3",
)]
#[case::anthropic_tool_use_name_and_input_without_id(
json!([
{"role": "user", "content": "fix the failing test"},
{"role": "assistant", "content": [
{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}},
]},
]),
r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#,
)]
#[case::anthropic_string_tool_result(
json!([{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"},
]}]),
"calc.py",
)]
#[case::anthropic_nested_text_tool_result_then_text(
json!([{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t1", "content": [
{"type": "text", "text": "a"},
{"type": "text", "text": "b"},
]},
{"type": "text", "text": "next"},
]}]),
"abnext",
)]
#[case::openai_tool_calls_in_order_before_tool_result(
json!([
{"role": "assistant", "content": "writing", "tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}},
{"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}},
]},
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
]),
r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}ok"#,
)]
#[case::malformed_tool_call_entries(
json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]),
r#"{"name":null,"arguments":null}"#,
)]
fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) {
assert_eq!(str_from_messages(messages.as_array().unwrap()), expected);
}
Expand Down Expand Up @@ -192,6 +231,23 @@ fn prompt_from_messages_reads_messages_only(
)]
#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))]
#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))]
#[case::responses_function_call(
None,
Some(json!([
{"role": "user", "content": "update the config"},
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"},
{"type": "function_call_output", "call_id": "c1", "output": "ok"},
])),
Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\nok"),
)]
#[case::responses_structured_function_call_output(
None,
Some(json!([
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]},
])),
Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\ndenied"),
)]
fn prompt_from_context_matches_python(
#[case] messages: Option<Value>,
#[case] input: Option<Value>,
Expand Down
10 changes: 5 additions & 5 deletions litellm/caching/qdrant_semantic_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
get_str_from_messages_with_tools,
)
from litellm.types.utils import EmbeddingResponse

Expand Down Expand Up @@ -286,7 +286,7 @@ def set_cache(self, key, value, **kwargs):

# get the prompt
messages: Final = kwargs["messages"]
prompt: Final = get_str_from_messages(messages)
prompt: Final = get_str_from_messages_with_tools(messages)

# create an embedding for prompt
embedding_response: Final = cast(
Expand Down Expand Up @@ -325,7 +325,7 @@ def get_cache(self, key, **kwargs):

# get the messages
messages: Final = kwargs["messages"]
prompt: Final = get_str_from_messages(messages)
prompt: Final = get_str_from_messages_with_tools(messages)

# convert to embedding
embedding_response: Final = cast(
Expand Down Expand Up @@ -400,7 +400,7 @@ async def async_set_cache(self, key, value, **kwargs):

# get the prompt
messages: Final = kwargs["messages"]
prompt: Final = get_str_from_messages(messages)
prompt: Final = get_str_from_messages_with_tools(messages)
embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))

# get the embedding
Expand Down Expand Up @@ -435,7 +435,7 @@ async def async_get_cache(self, key, **kwargs):

# get the messages
messages: Final = kwargs["messages"]
prompt: Final = get_str_from_messages(messages)
prompt: Final = get_str_from_messages_with_tools(messages)

embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))

Expand Down
32 changes: 22 additions & 10 deletions litellm/caching/redis_semantic_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages,
get_str_from_messages_with_tools,
tool_call_str,
)
from litellm.types.utils import EmbeddingResponse

Expand Down Expand Up @@ -263,7 +264,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None:
"""
messages: Final = kwargs.get("messages")
if messages:
return get_str_from_messages(messages)
return get_str_from_messages_with_tools(messages)

if "input" not in kwargs:
return None
Expand All @@ -275,7 +276,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None:

@classmethod
def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None:
value = cls._coerce_response_input_value(value)
value = cls._function_call_as_prompt(cls._coerce_response_input_value(value))
if value is None:
return

Expand All @@ -296,13 +297,7 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -
cls._collect_responses_input_text(content, prompt_parts)
return

for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
cls._collect_responses_text_fields(value, prompt_parts)
return

content = getattr(value, "content", None)
Expand All @@ -318,6 +313,23 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -
prompt_parts.append(stripped_text)
return

@classmethod
def _collect_responses_text_fields(cls, value: dict, prompt_parts: list[str]) -> None:
for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, (list, tuple)):
cls._collect_responses_input_text(text_value, prompt_parts)
return
if isinstance(text_value, str) and (stripped_text := text_value.strip()):
prompt_parts.append(stripped_text)
return

@staticmethod
def _function_call_as_prompt(value: object) -> object:
if isinstance(value, dict) and value.get("type") == "function_call":
return tool_call_str(value.get("name"), value.get("arguments"))
return value

@staticmethod
def _coerce_response_input_value(value: object) -> object:
model_dump: Final = getattr(value, "model_dump", None)
Expand Down
69 changes: 69 additions & 0 deletions litellm/litellm_core_utils/prompt_templates/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast

from pydantic import BaseModel
from typing_extensions import TypeIs # noqa: TID251 # narrows untyped message payloads without a runtime conversion

import litellm
from litellm import verbose_logger
from litellm.router_utils.batch_utils import InMemoryFile
Expand Down Expand Up @@ -192,6 +195,72 @@
return text


def get_str_from_messages_with_tools(messages: object) -> str:
"""
``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns
that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``)
produce different text
"""
return "".join(_message_str_with_tools(message) for message in _str_mappings(messages))


def tool_call_str(name: object, arguments: object) -> str:
return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}'


def _compact_json(value: object) -> str:
return json.dumps(value, separators=(",", ":"), default=str)


def _message_str_with_tools(message: Mapping[str, object]) -> str:
return (
_content_str_with_tools(message.get("content"))
+ "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls")))
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
+ extract_search_results_text(message.get("search_results"))
)


def _content_str_with_tools(content: object) -> str:
if isinstance(content, str):
return content
return "".join(_block_str_with_tools(block) for block in _str_mappings(content))


def _block_str_with_tools(block: Mapping[str, object]) -> str:
Comment thread
github-advanced-security[bot] marked this conversation as resolved.
Fixed
match block.get("type"):
case "tool_use":
return tool_call_str(block.get("name"), block.get("input"))
case "tool_result":
return _content_str_with_tools(block.get("content"))
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
case _:
text: Final = block.get("text")
return text if isinstance(text, str) else ""


def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str:
function: Final = _as_str_mapping(tool_call.get("function"))
if function is None:
return tool_call_str(None, None)
return tool_call_str(function.get("name"), function.get("arguments"))


def _str_mappings(values: object) -> Iterator[Mapping[str, object]]:
items: Final = values if isinstance(values, (list, tuple)) else ()
return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None)


def _as_str_mapping(value: object) -> Mapping[str, object] | None:
if isinstance(value, BaseModel):
return value.model_dump()
if _is_str_mapping(value):
return value
return None


def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str
return isinstance(value, Mapping)


def is_non_content_values_set(message: AllMessageValues) -> bool:
ignore_keys: Final = ["content", "role", "name"]
return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
Expand Down
1 change: 1 addition & 0 deletions tests/integration/_support/manifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"providers",
"streaming",
"messages_endpoint",
"caching",
"configuration",
"mcp",
"observability",
Expand Down
Loading
Loading