From f462fbeb2e59b6743aea8af5a5b92d2a8407faa8 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 16:43:47 +0000 Subject: [PATCH 01/29] fix(caching): keep tool calls and tool results in semantic cache prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 59 +++- litellm-rust/crates/cache/tests/semantic.rs | 48 +++ litellm/caching/qdrant_semantic_cache.py | 10 +- litellm/caching/redis_semantic_cache.py | 9 +- .../prompt_templates/common_utils.py | 63 ++++ tests/integration/_support/manifest.py | 1 + .../caching/test_semantic_cache_tool_turns.py | 277 ++++++++++++++++++ tests/integration/run.py | 2 +- .../unit/caching/test_redis_semantic_cache.py | 35 +++ ...ore_utils_prompt_templates_common_utils.py | 116 ++++++++ 10 files changed, 600 insertions(+), 20 deletions(-) create mode 100644 tests/integration/caching/test_semantic_cache_tool_turns.py diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index c88706a213b5..5b619151285e 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -1,6 +1,6 @@ //! The embedding and prompt contract every semantic backend shares. //! -//! Python's semantic caches all read their prompt through `get_str_from_messages`, and +//! Python's semantic caches all read their prompt through `get_str_from_messages_with_tools`, and //! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API //! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. @@ -8,7 +8,7 @@ use std::{future::Future, io}; use serde::Serialize; use serde_json::{ - Value, + Value, json, ser::{CharEscape, Formatter, Serializer}, }; @@ -83,26 +83,57 @@ impl Embedder for PreparedEmbedding { } } -/// `get_str_from_messages`: every message's text content followed by its search results. +/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, +/// then its OpenAI `tool_calls`, then its search results. pub fn str_from_messages(messages: &[Value]) -> String { let mut text = String::new(); for message in messages.iter().filter_map(Value::as_object) { - match message.get("content") { - Some(Value::String(content)) => text.push_str(content), - Some(Value::Array(parts)) => { - for part in parts { - if let Some(part_text) = part.get("text").and_then(Value::as_str) { - text.push_str(part_text); - } - } + push_content_text(&mut text, message.get("content")); + if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { + for tool_call in tool_calls.iter().filter_map(Value::as_object) { + let function = tool_call.get("function"); + text.push_str(&tool_call_json( + function.and_then(|function| function.get("name")), + function.and_then(|function| function.get("arguments")), + )); } - _ => {} } push_search_results_text(&mut text, message.get("search_results")); } text } +/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. +fn push_content_text(text: &mut String, content: Option<&Value>) { + match content { + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(blocks)) => { + for block in blocks.iter().filter_map(Value::as_object) { + match block.get("type").and_then(Value::as_str) { + Some("tool_use") => { + text.push_str(&tool_call_json(block.get("name"), block.get("input"))); + } + Some("tool_result") => push_content_text(text, block.get("content")), + _ => { + if let Some(block_text) = block.get("text").and_then(Value::as_str) { + text.push_str(block_text); + } + } + } + } + } + _ => {} + } +} + +/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. +fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { + compact_json(&json!({ + "name": name.unwrap_or(&Value::Null), + "arguments": arguments.unwrap_or(&Value::Null), + })) +} + /// The messages prompt Qdrant embeds: `None` when the request carries no messages. pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { let messages = context.messages.as_ref()?.as_array()?; @@ -159,6 +190,10 @@ fn collect_input_text(value: &Value, parts: &mut Vec) { } } Value::Object(map) => { + if map.get("type").and_then(Value::as_str) == Some("function_call") { + parts.push(tool_call_json(map.get("name"), map.get("arguments"))); + return; + } if let Some(content) = map.get("content").filter(|content| !content.is_null()) { collect_input_text(content, parts); return; diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index 97a552a8010a..ab6b904af1a6 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -100,6 +100,45 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), "false3", )] +#[case::anthropic_tool_use_name_and_input_without_id( + json!([ + {"role": "user", "content": "fix the failing test"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, + ]}, + ]), + r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, +)] +#[case::anthropic_string_tool_result( + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, + ]}]), + "calc.py", +)] +#[case::anthropic_nested_text_tool_result_then_text( + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b"}, + ]}, + {"type": "text", "text": "next"}, + ]}]), + "abnext", +)] +#[case::openai_tool_calls_in_order_before_tool_result( + json!([ + {"role": "assistant", "content": "writing", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, + {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, + ]}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + ]), + r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}ok"#, +)] +#[case::malformed_tool_call_entries( + json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), + r#"{"name":null,"arguments":null}"#, +)] fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); } @@ -192,6 +231,15 @@ fn prompt_from_messages_reads_messages_only( )] #[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] #[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] +#[case::responses_function_call( + None, + Some(json!([ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ])), + Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\nok"), +)] fn prompt_from_context_matches_python( #[case] messages: Option, #[case] input: Option, diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index b99023c07fd4..8fe03131760e 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -24,7 +24,7 @@ ) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages, + get_str_from_messages_with_tools, ) from litellm.types.utils import EmbeddingResponse @@ -286,7 +286,7 @@ def set_cache(self, key, value, **kwargs): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_str_from_messages_with_tools(messages) # create an embedding for prompt embedding_response: Final = cast( @@ -325,7 +325,7 @@ def get_cache(self, key, **kwargs): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_str_from_messages_with_tools(messages) # convert to embedding embedding_response: Final = cast( @@ -400,7 +400,7 @@ async def async_set_cache(self, key, value, **kwargs): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_str_from_messages_with_tools(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding @@ -435,7 +435,7 @@ async def async_get_cache(self, key, **kwargs): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages(messages) + prompt: Final = get_str_from_messages_with_tools(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index d4c815e15b7a..1cb1b457cd5d 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -21,7 +21,8 @@ from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages, + get_str_from_messages_with_tools, + tool_call_str, ) from litellm.types.utils import EmbeddingResponse @@ -263,7 +264,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: """ messages: Final = kwargs.get("messages") if messages: - return get_str_from_messages(messages) + return get_str_from_messages_with_tools(messages) if "input" not in kwargs: return None @@ -291,6 +292,10 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - return if isinstance(value, dict): + if value.get("type") == "function_call": + prompt_parts.append(tool_call_str(value.get("name"), value.get("arguments"))) + return + content = value.get("content") if content is not None: cls._collect_responses_input_text(content, prompt_parts) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 3b6827375f31..90e54750fd1f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,6 +13,9 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast +from pydantic import BaseModel +from typing_extensions import TypeIs # noqa: TID251 # narrows untyped message payloads without a runtime conversion + import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -192,6 +195,66 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text +def get_str_from_messages_with_tools(messages: object) -> str: + """ + ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns + that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) + produce different text + """ + return "".join(_message_str_with_tools(message) for message in _str_mappings(messages)) + + +def tool_call_str(name: object, arguments: object) -> str: + return json.dumps({"name": name, "arguments": arguments}, separators=(",", ":"), default=str) + + +def _message_str_with_tools(message: Mapping[str, object]) -> str: + return ( + _content_str_with_tools(message.get("content")) + + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) + + extract_search_results_text(message.get("search_results")) + ) + + +def _content_str_with_tools(content: object) -> str: + if isinstance(content, str): + return content + return "".join(_block_str_with_tools(block) for block in _str_mappings(content)) + + +def _block_str_with_tools(block: Mapping[str, object]) -> str: + match block.get("type"): + case "tool_use": + return tool_call_str(block.get("name"), block.get("input")) + case "tool_result": + return _content_str_with_tools(block.get("content")) + case _: + text: Final = block.get("text") + return text if isinstance(text, str) else "" + + +def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: + function: Final = _as_str_mapping(tool_call.get("function")) or {} + return tool_call_str(function.get("name"), function.get("arguments")) + + +def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: + items: Final = values if isinstance(values, (list, tuple)) else () + return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) + + +def _as_str_mapping(value: object) -> Mapping[str, object] | None: + if isinstance(value, BaseModel): + return value.model_dump() + if _is_str_mapping(value): + return value + return None + + +def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str + return isinstance(value, Mapping) + + def is_non_content_values_set(message: AllMessageValues) -> bool: ignore_keys: Final = ["content", "role", "name"] return any(message.get(key, None) is not None for key in message if key not in ignore_keys) diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index a4a86a215684..a37ae94c2ed9 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -11,6 +11,7 @@ "providers", "streaming", "messages_endpoint", + "caching", "configuration", "mcp", "observability", diff --git a/tests/integration/caching/test_semantic_cache_tool_turns.py b/tests/integration/caching/test_semantic_cache_tool_turns.py new file mode 100644 index 000000000000..6c9a0b060fc3 --- /dev/null +++ b/tests/integration/caching/test_semantic_cache_tool_turns.py @@ -0,0 +1,277 @@ +import hashlib +import json +import math +import threading +import uuid +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_VECTOR_SIZE: Final = 16 +_CHAT_MODEL: Final = "semantic-chat" +_CLAUDE_MODEL: Final = "semantic-claude" +_EMBEDDING_MODEL: Final = "semantic-embedder" +_COLLECTION: Final = "semantic-tool-turns" + + +def _vector(text: str) -> tuple[float, ...]: + digest: Final = hashlib.sha256(text.encode()).digest() + return tuple((byte - 127.5) / 127.5 for byte in digest[:_VECTOR_SIZE]) + + +def _cosine(left: Sequence[float], right: Sequence[float]) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + return dot / (math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))) + + +def _answer(body: Mapping[str, JsonValue]) -> str: + return "answer-" + hashlib.sha256(json.dumps(body["messages"], sort_keys=True).encode()).hexdigest()[:16] + + +@dataclass(slots=True) +class _Peer: + """Embeddings, chat, Anthropic Messages and a Qdrant collection on one owned socket""" + + lock: threading.Lock = field(default_factory=threading.Lock) + points: list[Mapping[str, JsonValue]] = field(default_factory=list) # mutable-ok: the Qdrant collection + embedded: list[str] = field(default_factory=list) # mutable-ok: every prompt the proxy embedded + answered: list[str] = field(default_factory=list) # mutable-ok: every completion the provider served + + def stored(self) -> int: + with self.lock: + return len(self.points) + + def respond(self, request: Request) -> Reply: + path: Final = urlsplit(request.target).path + body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} + collection: Final = f"/qdrant/collections/{_COLLECTION}" + if request.method == "GET" and path == f"{collection}/exists": + return self._json({"result": {"exists": False}, "status": "ok"}) + if request.method in {"GET", "PUT"} and path in {collection, f"{collection}/index"}: + return self._json({"result": True, "status": "ok"}) + if request.method == "PUT" and path == f"{collection}/points": + with self.lock: + self.points.extend(_JSON_OBJECT.validate_python(point) for point in body["points"]) + return self._json({"result": {"status": "completed"}, "status": "ok"}) + if request.method == "POST" and path == f"{collection}/points/search": + return self._json({"result": self._search(body), "status": "ok"}) + if request.method == "GET" and path == "/v1/models": + return self._json({"object": "list", "data": []}) + if request.method == "POST" and path == "/v1/embeddings": + text: Final = str(body["input"]) + with self.lock: + self.embedded.append(text) + return self._json( + { + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": list(_vector(text))}], + "usage": {"prompt_tokens": 4, "total_tokens": 4}, + } + ) + if request.method == "POST" and path == "/v1/chat/completions": + return self._json(self._chat_reply(_answer(body))) + if request.method == "POST" and path == "/v1/messages": + return self._json(self._messages_reply(_answer(body))) + raise AssertionError(f"unexpected peer request {request.method} {request.target}") + + def _search(self, body: Mapping[str, JsonValue]) -> list[JsonValue]: + query: Final = [float(str(value)) for value in _list(body["vector"])] + key: Final = _JSON_OBJECT.validate_python( + _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_python(body["filter"])["must"])[0])["match"] + )["value"] + with self.lock: + scoped: Final = [ + point + for point in self.points + if _JSON_OBJECT.validate_python(point["payload"])["litellm_cache_key"] == key + ] + ranked: Final = sorted( + ( + { + "id": point["id"], + "score": _cosine(query, [float(str(value)) for value in _list(point["vector"])]), + "payload": point["payload"], + } + for point in scoped + ), + key=lambda hit: -float(str(hit["score"])), + ) + return list(ranked[:1]) + + def _chat_reply(self, answer: str) -> Mapping[str, JsonValue]: + with self.lock: + self.answered.append(answer) + return { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1789788253, + "model": "gpt-5.4-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + + def _messages_reply(self, answer: str) -> Mapping[str, JsonValue]: + with self.lock: + self.answered.append(answer) + return { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": answer}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 2}, + } + + @staticmethod + def _json(value: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(value).encode()) + + +def _list(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +@dataclass(frozen=True, slots=True) +class _SemanticProxy: + gateway: Gateway + peer: _Peer + + +@pytest.fixture(scope="module") +def semantic_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SemanticProxy]: + peer: Final = _Peer() + directory: Final = tmp_path_factory.mktemp("semantic-tool-turns") + with gateway_from_environment() as gateway, wire_server(peer.respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": _CHAT_MODEL, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_base": f"{wire.url}/v1", "api_key": "k"}, + }, + { + "model_name": _CLAUDE_MODEL, + "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_base": wire.url, "api_key": "k"}, + }, + { + "model_name": _EMBEDDING_MODEL, + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_base": f"{wire.url}/v1", + "api_key": "k", + }, + }, + ] + config["litellm_settings"]["cache_params"] = { + "type": "qdrant-semantic", + "qdrant_api_base": f"{wire.url}/qdrant", + "qdrant_collection_name": _COLLECTION, + "qdrant_semantic_cache_embedding_model": _EMBEDDING_MODEL, + "qdrant_semantic_cache_vector_size": _VECTOR_SIZE, + "qdrant_quantization_config": "binary", + "similarity_threshold": 0.99, + } + path: Final = directory / "semantic_tool_turns.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, directory, {}, config=path) as candidate: + yield _SemanticProxy(candidate, peer) + + +def _send_turns(proxy: _SemanticProxy, route: str, model: str, turns: Sequence[list[JsonValue]]) -> list[str]: + def reply_text(turn: list[JsonValue]) -> str: + stored_before: Final = proxy.peer.stored() + answered_before: Final = len(proxy.peer.answered) + body: Final[dict[str, JsonValue]] = {"model": model, "max_tokens": 16, "messages": turn} + response: Final = proxy.gateway.request("POST", route, body) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + if len(proxy.peer.answered) > answered_before: + eventually(proxy.peer.stored, lambda count: count > stored_before) + if route == "/v1/messages": + return str(_JSON_OBJECT.validate_python(_list(payload["content"])[0])["text"]) + choice: Final = _JSON_OBJECT.validate_python(_list(payload["choices"])[0]) + return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) + + return [reply_text(turn) for turn in turns] + + +def test_claude_code_tool_turns_on_messages_are_not_served_the_first_turn_answer( + semantic_proxy: _SemanticProxy, +) -> None: + task: Final[JsonValue] = {"role": "user", "content": f"fix the failing test {uuid.uuid4().hex}"} + list_files: Final[list[JsonValue]] = [ + task, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, + ] + read_file: Final[list[JsonValue]] = [ + *list_files, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_2", "name": "Read", "input": {"file_path": "calc.py"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_2", + "content": [{"type": "text", "text": "def add(a, b): return a - b"}], + } + ], + }, + ] + + answers: Final = _send_turns(semantic_proxy, "/v1/messages", _CLAUDE_MODEL, ([task], list_files, read_file)) + + assert len(set(answers)) == 3, f"a later tool turn replayed an earlier cached answer: {answers}" + assert answers == semantic_proxy.peer.answered[-3:], semantic_proxy.peer.answered + + +def test_openai_agent_loop_tool_calls_on_chat_completions_are_not_served_a_cached_answer( + semantic_proxy: _SemanticProxy, +) -> None: + task: Final[JsonValue] = {"role": "user", "content": f"update both config files {uuid.uuid4().hex}"} + + def wrote(path: str) -> list[JsonValue]: + return [ + task, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "write_file", "arguments": json.dumps({"path": path})}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, + ] + + answers: Final = _send_turns( + semantic_proxy, "/v1/chat/completions", _CHAT_MODEL, (wrote("a.yaml"), wrote("b.yaml")) + ) + + assert len(set(answers)) == 2, f"a different tool call replayed an earlier cached answer: {answers}" + assert answers == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered diff --git a/tests/integration/run.py b/tests/integration/run.py index 19bce35f5421..2e82fa4539d7 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -14,7 +14,7 @@ "management": ("management", "authorization", "configuration"), "accounting": ("pricing", "spend"), "database": ("database",), - "providers": ("providers", "routing", "streaming", "messages_endpoint"), + "providers": ("providers", "routing", "streaming", "messages_endpoint", "caching"), "extensions": ("observability", "compatibility"), "mcp": ("mcp",), "sdk": ("sdk",), diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 461689165bb1..1e4a6c2bdc66 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -579,6 +579,41 @@ def test_redis_semantic_cache_prompt_extraction_prefers_messages(): assert prompt == "message prompt" +def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + def turn(command: str) -> list[dict[str, object]]: + return [ + {"role": "user", "content": "fix the failing test"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": command}}], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]}, + ] + + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}ok' + ) + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( + 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}ok' + ) + + +def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path":"a.yaml"}'}, + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ] + ) + + assert prompt == 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\nok' + + def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): from litellm.caching.redis_semantic_cache import RedisSemanticCache diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 0375ff14852a..4c1af95f4fe2 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -7,6 +7,8 @@ import pytest +from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message + from litellm.litellm_core_utils.prompt_templates.common_utils import ( ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, @@ -16,6 +18,8 @@ encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, + get_str_from_messages, + get_str_from_messages_with_tools, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, @@ -2004,3 +2008,115 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ) assert merged == [{"role": "system"}, {"role": "user", "content": "Hi"}] + + +_TASK: Final = {"role": "user", "content": "fix the failing test"} + + +@pytest.mark.parametrize( + ("messages", "expected"), + [ + pytest.param( + [ + _TASK, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], + }, + ], + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', + id="anthropic-tool-use-name-and-input-without-id", + ), + pytest.param( + [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], + "fix the failing testcalc.py", + id="anthropic-string-tool-result", + ), + pytest.param( + [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "t1", + "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], + }, + {"type": "text", "text": "next"}, + ], + } + ], + "abnext", + id="anthropic-nested-text-tool-result-then-text", + ), + pytest.param( + [ + _TASK, + { + "role": "assistant", + "content": "writing", + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "write", "arguments": '{"path": "a"}'}}, + {"id": "c2", "type": "function", "function": {"name": "write", "arguments": '{"path": "b"}'}}, + ], + }, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + ], + 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' + '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}ok', + id="openai-tool-calls-in-order-before-tool-result", + ), + pytest.param( + [ + Message( + content=None, + tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], + ) + ], + '{"name":"read","arguments":"{}"}', + id="openai-response-message-object", + ), + pytest.param( + [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], + '{"name":null,"arguments":null}', + id="malformed-tool-call-entries", + ), + ], +) +def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: + assert get_str_from_messages_with_tools(messages) == expected + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), + pytest.param( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this"}, + ], + } + ], + id="text-and-image-parts", + ), + pytest.param( + [ + { + "role": "tool", + "tool_call_id": "c1", + "content": "small", + "search_results": [{"source": "s", "title": "t"}], + } + ], + id="search-results", + ), + pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), + ], +) +def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: + assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures From 8b08e07f188f65520c6b160035cce293c837b734 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 16:53:53 +0000 Subject: [PATCH 02/29] refactor(caching): keep semantic tool prompt helpers within lint budgets Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_semantic_cache.py | 12 +++++++----- .../prompt_templates/common_utils.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 1cb1b457cd5d..bac58026f185 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -276,7 +276,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: @classmethod def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: - value = cls._coerce_response_input_value(value) + value = cls._function_call_as_prompt(cls._coerce_response_input_value(value)) if value is None: return @@ -292,10 +292,6 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - return if isinstance(value, dict): - if value.get("type") == "function_call": - prompt_parts.append(tool_call_str(value.get("name"), value.get("arguments"))) - return - content = value.get("content") if content is not None: cls._collect_responses_input_text(content, prompt_parts) @@ -323,6 +319,12 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - prompt_parts.append(stripped_text) return + @staticmethod + def _function_call_as_prompt(value: object) -> object: + if isinstance(value, dict) and value.get("type") == "function_call": + return tool_call_str(value.get("name"), value.get("arguments")) + return value + @staticmethod def _coerce_response_input_value(value: object) -> object: model_dump: Final = getattr(value, "model_dump", None) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 90e54750fd1f..d7fe18496703 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -205,7 +205,11 @@ def get_str_from_messages_with_tools(messages: object) -> str: def tool_call_str(name: object, arguments: object) -> str: - return json.dumps({"name": name, "arguments": arguments}, separators=(",", ":"), default=str) + return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), default=str) def _message_str_with_tools(message: Mapping[str, object]) -> str: @@ -234,7 +238,9 @@ def _block_str_with_tools(block: Mapping[str, object]) -> str: def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: - function: Final = _as_str_mapping(tool_call.get("function")) or {} + function: Final = _as_str_mapping(tool_call.get("function")) + if function is None: + return tool_call_str(None, None) return tool_call_str(function.get("name"), function.get("arguments")) From 8bf443997baa12464a1a72721915ce2792df6971 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 17:09:18 +0000 Subject: [PATCH 03/29] fix(caching): keep structured function_call_output text in semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 11 ++++++---- litellm-rust/crates/cache/tests/semantic.rs | 8 +++++++ litellm/caching/redis_semantic_cache.py | 3 +++ .../unit/caching/test_redis_semantic_cache.py | 21 +++++++++++++++++++ 4 files changed, 39 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 5b619151285e..b27ab269edd5 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -199,10 +199,13 @@ fn collect_input_text(value: &Value, parts: &mut Vec) { return; } for key in ["text", "output", "input_text", "output_text"] { - if let Some(Value::String(text)) = map.get(key) - && push_trimmed(text, parts) - { - return; + match map.get(key) { + Some(nested @ Value::Array(_)) => { + collect_input_text(nested, parts); + return; + } + Some(Value::String(text)) if push_trimmed(text, parts) => return, + _ => {} } } } diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index ab6b904af1a6..a29844bc2410 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -240,6 +240,14 @@ fn prompt_from_messages_reads_messages_only( ])), Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\nok"), )] +#[case::responses_structured_function_call_output( + None, + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, + ])), + Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\ndenied"), +)] fn prompt_from_context_matches_python( #[case] messages: Option, #[case] input: Option, diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index bac58026f185..cd0f0770eec3 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -299,6 +299,9 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - for text_key in ("text", "output", "input_text", "output_text"): text_value = value.get(text_key) + if isinstance(text_value, (list, tuple)): + cls._collect_responses_input_text(text_value, prompt_parts) + return if isinstance(text_value, str): stripped_text = text_value.strip() if stripped_text: diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 1e4a6c2bdc66..38700dce62c2 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -614,6 +614,27 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() assert prompt == 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\nok' +def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_outputs(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + def prompt_for(output_text: str) -> str | None: + return RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "write hello"}]}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path":"a.txt"}'}, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": output_text}], + }, + ] + ) + + expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' + assert prompt_for("wrote 5 bytes") == f"write hello\n{expected_call}\nwrote 5 bytes" + assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") + + def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): from litellm.caching.redis_semantic_cache import RedisSemanticCache From 516586008c82357c7a5f02dea8f5bdfdc7b5a4aa Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 17:21:59 +0000 Subject: [PATCH 04/29] refactor(caching): split Responses text-field collection to stay within complexity budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_semantic_cache.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index cd0f0770eec3..9968297f01d4 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -297,16 +297,7 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - cls._collect_responses_input_text(content, prompt_parts) return - for text_key in ("text", "output", "input_text", "output_text"): - text_value = value.get(text_key) - if isinstance(text_value, (list, tuple)): - cls._collect_responses_input_text(text_value, prompt_parts) - return - if isinstance(text_value, str): - stripped_text = text_value.strip() - if stripped_text: - prompt_parts.append(stripped_text) - return + cls._collect_responses_text_fields(value, prompt_parts) return content = getattr(value, "content", None) @@ -322,6 +313,17 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - prompt_parts.append(stripped_text) return + @classmethod + def _collect_responses_text_fields(cls, value: dict, prompt_parts: list[str]) -> None: + for text_key in ("text", "output", "input_text", "output_text"): + text_value = value.get(text_key) + if isinstance(text_value, (list, tuple)): + cls._collect_responses_input_text(text_value, prompt_parts) + return + if isinstance(text_value, str) and (stripped_text := text_value.strip()): + prompt_parts.append(stripped_text) + return + @staticmethod def _function_call_as_prompt(value: object) -> object: if isinstance(value, dict) and value.get("type") == "function_call": From 19bc489cee6c876b54d4b78cb6461c54288d2421 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 17:57:55 +0000 Subject: [PATCH 05/29] fix(caching): tag each tool result with the position of the call it answers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 95 ++++++++++++++++--- litellm-rust/crates/cache/tests/semantic.rs | 54 ++++++++++- litellm/caching/redis_semantic_cache.py | 46 ++++++--- .../prompt_templates/common_utils.py | 56 ++++++++--- .../caching/test_semantic_cache_tool_turns.py | 16 ++-- .../unit/caching/test_redis_semantic_cache.py | 30 +++++- ...ore_utils_prompt_templates_common_utils.py | 79 ++++++++++++++- 7 files changed, 322 insertions(+), 54 deletions(-) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index b27ab269edd5..990f5e47aa85 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -4,7 +4,7 @@ //! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API //! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. -use std::{future::Future, io}; +use std::{collections::HashMap, future::Future, io}; use serde::Serialize; use serde_json::{ @@ -84,11 +84,35 @@ impl Embedder for PreparedEmbedding { } /// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, -/// then its OpenAI `tool_calls`, then its search results. +/// then its OpenAI `tool_calls`, then its search results. Each tool result is tagged with the +/// position of the call it answers. pub fn str_from_messages(messages: &[Value]) -> String { + let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); + let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { + let block_ids = message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) + .filter_map(|block| block.get("id")); + let tool_call_ids = message + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tool_call| tool_call.get("id")); + block_ids.chain(tool_call_ids) + })); let mut text = String::new(); - for message in messages.iter().filter_map(Value::as_object) { - push_content_text(&mut text, message.get("content")); + for message in messages { + if message.get("role").and_then(Value::as_str) == Some("tool") { + text.push_str(&tool_result_tag( + message.get("tool_call_id"), + &call_ordinals, + )); + } + push_content_text(&mut text, message.get("content"), &call_ordinals); if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { for tool_call in tool_calls.iter().filter_map(Value::as_object) { let function = tool_call.get("function"); @@ -104,7 +128,11 @@ pub fn str_from_messages(messages: &[Value]) -> String { } /// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. -fn push_content_text(text: &mut String, content: Option<&Value>) { +fn push_content_text( + text: &mut String, + content: Option<&Value>, + call_ordinals: &HashMap<&str, usize>, +) { match content { Some(Value::String(content)) => text.push_str(content), Some(Value::Array(blocks)) => { @@ -113,7 +141,10 @@ fn push_content_text(text: &mut String, content: Option<&Value>) { Some("tool_use") => { text.push_str(&tool_call_json(block.get("name"), block.get("input"))); } - Some("tool_result") => push_content_text(text, block.get("content")), + Some("tool_result") => { + text.push_str(&tool_result_tag(block.get("tool_use_id"), call_ordinals)); + push_content_text(text, block.get("content"), call_ordinals); + } _ => { if let Some(block_text) = block.get("text").and_then(Value::as_str) { text.push_str(block_text); @@ -126,6 +157,26 @@ fn push_content_text(text: &mut String, content: Option<&Value>) { } } +/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. +fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { + let mut ordinals = HashMap::new(); + for call_id in call_ids.filter_map(Value::as_str) { + let next = ordinals.len() + 1; + ordinals.entry(call_id).or_insert(next); + } + ordinals +} + +/// `tool_result_str`: `{"result_of_call":N}` for a result answering a known call, else empty. +fn tool_result_tag(call_id: Option<&Value>, call_ordinals: &HashMap<&str, usize>) -> String { + call_id + .and_then(Value::as_str) + .and_then(|call_id| call_ordinals.get(call_id)) + .map_or_else(String::new, |ordinal| { + format!("{{\"result_of_call\":{ordinal}}}") + }) +} + /// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { compact_json(&json!({ @@ -149,8 +200,16 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { return Some(str_from_messages(messages)); } let input = context.input.as_ref()?; + let call_ordinals = tool_call_ordinals( + input + .as_array() + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) + .filter_map(|item| item.get("call_id")), + ); let mut parts = Vec::new(); - collect_input_text(input, &mut parts); + collect_input_text(input, &mut parts, &call_ordinals); let prompt = python_strip(&parts.join("\n")).to_owned(); (!prompt.is_empty()).then_some(prompt) } @@ -179,14 +238,18 @@ fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { } } -fn collect_input_text(value: &Value, parts: &mut Vec) { +fn collect_input_text( + value: &Value, + parts: &mut Vec, + call_ordinals: &HashMap<&str, usize>, +) { match value { Value::String(text) => { push_trimmed(text, parts); } Value::Array(items) => { for item in items { - collect_input_text(item, parts); + collect_input_text(item, parts, call_ordinals); } } Value::Object(map) => { @@ -194,14 +257,24 @@ fn collect_input_text(value: &Value, parts: &mut Vec) { parts.push(tool_call_json(map.get("name"), map.get("arguments"))); return; } + if map.get("type").and_then(Value::as_str) == Some("function_call_output") { + let tag = tool_result_tag(map.get("call_id"), call_ordinals); + if !tag.is_empty() { + parts.push(tag); + if let Some(output) = map.get("output") { + collect_input_text(output, parts, call_ordinals); + } + return; + } + } if let Some(content) = map.get("content").filter(|content| !content.is_null()) { - collect_input_text(content, parts); + collect_input_text(content, parts, call_ordinals); return; } for key in ["text", "output", "input_text", "output_text"] { match map.get(key) { Some(nested @ Value::Array(_)) => { - collect_input_text(nested, parts); + collect_input_text(nested, parts, call_ordinals); return; } Some(Value::String(text)) if push_trimmed(text, parts) => return, diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index a29844bc2410..7ede4948044d 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -133,7 +133,40 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ]), - r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}ok"#, + r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1}ok"#, +)] +#[case::anthropic_parallel_tool_results_tagged_with_their_call( + json!([ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, + {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, + {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, + ]}, + ]), + r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2}B{"result_of_call":1}A"#, +)] +#[case::reused_call_ids_keep_first_position( + json!([ + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, + ]}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + ]), + r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2}C"#, +)] +#[case::unknown_call_ids_left_untagged( + json!([ + {"role": "tool", "tool_call_id": "c9", "content": "ok"}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, + ]), + "okdone", )] #[case::malformed_tool_call_entries( json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), @@ -238,7 +271,7 @@ fn prompt_from_messages_reads_messages_only( {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}, ])), - Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\nok"), + Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1}\nok"), )] #[case::responses_structured_function_call_output( None, @@ -246,7 +279,22 @@ fn prompt_from_messages_reads_messages_only( {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, ])), - Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\ndenied"), + Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1}\ndenied"), +)] +#[case::responses_parallel_outputs_tagged_with_their_call( + None, + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, + {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, + {"type": "function_call_output", "call_id": "c2", "output": "B"}, + {"type": "function_call_output", "call_id": "c1", "output": "A"}, + ])), + Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2}\nB\n{\"result_of_call\":1}\nA"), +)] +#[case::responses_unknown_call_id_output_untagged( + None, + Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), + Some("orphan"), )] fn prompt_from_context_matches_python( #[case] messages: Option, diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 9968297f01d4..88f5acf026e5 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,7 +22,9 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages_with_tools, + tool_call_ordinals, tool_call_str, + tool_result_str, ) from litellm.types.utils import EmbeddingResponse @@ -269,14 +271,17 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: if "input" not in kwargs: return None + responses_input: Final = kwargs.get("input") prompt_parts: Final[list[str]] = [] - cls._collect_responses_input_text(kwargs.get("input"), prompt_parts) + cls._collect_responses_input_text(responses_input, prompt_parts, cls._responses_call_ordinals(responses_input)) prompt: Final = "\n".join(prompt_parts).strip() return prompt or None @classmethod - def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: - value = cls._function_call_as_prompt(cls._coerce_response_input_value(value)) + def _collect_responses_input_text( + cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: + value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) if value is None: return @@ -288,21 +293,21 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - if isinstance(value, (list, tuple)): for item in value: - cls._collect_responses_input_text(item, prompt_parts) + cls._collect_responses_input_text(item, prompt_parts, call_ordinals) return if isinstance(value, dict): content = value.get("content") if content is not None: - cls._collect_responses_input_text(content, prompt_parts) + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) return - cls._collect_responses_text_fields(value, prompt_parts) + cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) return content = getattr(value, "content", None) if content is not None: - cls._collect_responses_input_text(content, prompt_parts) + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) return for text_key in ("text", "output", "input_text", "output_text"): @@ -314,21 +319,38 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - return @classmethod - def _collect_responses_text_fields(cls, value: dict, prompt_parts: list[str]) -> None: + def _collect_responses_text_fields( + cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: for text_key in ("text", "output", "input_text", "output_text"): text_value = value.get(text_key) if isinstance(text_value, (list, tuple)): - cls._collect_responses_input_text(text_value, prompt_parts) + cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) return if isinstance(text_value, str) and (stripped_text := text_value.strip()): prompt_parts.append(stripped_text) return + @classmethod + def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: + items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () + dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) + return tool_call_ordinals( + item.get("call_id") + for item in dumped_items + if isinstance(item, dict) and item.get("type") == "function_call" + ) + @staticmethod - def _function_call_as_prompt(value: object) -> object: - if isinstance(value, dict) and value.get("type") == "function_call": + def _function_call_as_prompt(value: object, call_ordinals: Mapping[str, int]) -> object: + if not isinstance(value, dict): + return value + if value.get("type") == "function_call": return tool_call_str(value.get("name"), value.get("arguments")) - return value + result_tag: Final = ( + tool_result_str(value.get("call_id"), call_ordinals) if value.get("type") == "function_call_output" else "" + ) + return (result_tag, value.get("output")) if result_tag else value @staticmethod def _coerce_response_input_value(value: object) -> object: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index d7fe18496703..dddb9f3b4682 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -199,42 +199,68 @@ def get_str_from_messages_with_tools(messages: object) -> str: """ ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) - produce different text + produce different text. Each result is tagged with the position of the call it answers, since call ids are + random per session """ - return "".join(_message_str_with_tools(message) for message in _str_mappings(messages)) + message_mappings: Final = tuple(_str_mappings(messages)) + call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) + return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) def tool_call_str(name: object, arguments: object) -> str: return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' +def tool_result_str(call_id: object, call_ordinals: Mapping[str, int]) -> str: + ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None + return "" if ordinal is None else f'{{"result_of_call":{ordinal}}}' + + +def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: + string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) + return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) + + def _compact_json(value: object) -> str: return json.dumps(value, separators=(",", ":"), default=str) -def _message_str_with_tools(message: Mapping[str, object]) -> str: +def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: + for message in messages: + yield from ( + block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" + ) + yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) + + +def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + result_tag: Final = ( + tool_result_str(message.get("tool_call_id"), call_ordinals) if message.get("role") == "tool" else "" + ) return ( - _content_str_with_tools(message.get("content")) + result_tag + + _content_str_with_tools(message.get("content"), call_ordinals) + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) + extract_search_results_text(message.get("search_results")) ) -def _content_str_with_tools(content: object) -> str: +def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: if isinstance(content, str): return content - return "".join(_block_str_with_tools(block) for block in _str_mappings(content)) + return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) -def _block_str_with_tools(block: Mapping[str, object]) -> str: - match block.get("type"): - case "tool_use": - return tool_call_str(block.get("name"), block.get("input")) - case "tool_result": - return _content_str_with_tools(block.get("content")) - case _: - text: Final = block.get("text") - return text if isinstance(text, str) else "" +def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + block_type: Final = block.get("type") + if block_type == "tool_use": + return tool_call_str(block.get("name"), block.get("input")) + if block_type == "tool_result": + return tool_result_str(block.get("tool_use_id"), call_ordinals) + _content_str_with_tools( + block.get("content"), call_ordinals + ) + text: Final = block.get("text") + return text if isinstance(text, str) else "" def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: diff --git a/tests/integration/caching/test_semantic_cache_tool_turns.py b/tests/integration/caching/test_semantic_cache_tool_turns.py index 6c9a0b060fc3..5d6e6a956a16 100644 --- a/tests/integration/caching/test_semantic_cache_tool_turns.py +++ b/tests/integration/caching/test_semantic_cache_tool_turns.py @@ -241,10 +241,13 @@ def test_claude_code_tool_turns_on_messages_are_not_served_the_first_turn_answer }, ] - answers: Final = _send_turns(semantic_proxy, "/v1/messages", _CLAUDE_MODEL, ([task], list_files, read_file)) + answers: Final = _send_turns( + semantic_proxy, "/v1/messages", _CLAUDE_MODEL, ([task], list_files, read_file, list_files) + ) - assert len(set(answers)) == 3, f"a later tool turn replayed an earlier cached answer: {answers}" - assert answers == semantic_proxy.peer.answered[-3:], semantic_proxy.peer.answered + assert len(set(answers[:3])) == 3, f"a later tool turn replayed an earlier cached answer: {answers}" + assert answers[:3] == semantic_proxy.peer.answered[-3:], semantic_proxy.peer.answered + assert answers[3] == answers[1], f"a repeated tool turn missed the cache: {answers}" def test_openai_agent_loop_tool_calls_on_chat_completions_are_not_served_a_cached_answer( @@ -270,8 +273,9 @@ def wrote(path: str) -> list[JsonValue]: ] answers: Final = _send_turns( - semantic_proxy, "/v1/chat/completions", _CHAT_MODEL, (wrote("a.yaml"), wrote("b.yaml")) + semantic_proxy, "/v1/chat/completions", _CHAT_MODEL, (wrote("a.yaml"), wrote("b.yaml"), wrote("a.yaml")) ) - assert len(set(answers)) == 2, f"a different tool call replayed an earlier cached answer: {answers}" - assert answers == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered + assert len(set(answers[:2])) == 2, f"a different tool call replayed an earlier cached answer: {answers}" + assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered + assert answers[2] == answers[0], f"a repeated tool call missed the cache: {answers}" diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 38700dce62c2..6ca4481648b6 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -593,10 +593,10 @@ def turn(command: str) -> list[dict[str, object]]: ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}ok' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1}ok' ) assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}ok' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1}ok' ) @@ -611,7 +611,9 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ] ) - assert prompt == 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\nok' + assert prompt == ( + 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1}\nok' + ) def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_outputs(): @@ -631,10 +633,30 @@ def prompt_for(output_text: str) -> str | None: ) expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' - assert prompt_for("wrote 5 bytes") == f"write hello\n{expected_call}\nwrote 5 bytes" + assert prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1}}\nwrote 5 bytes' assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") +def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | None: + return RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, + {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, + {"type": "function_call_output", "call_id": first_output_call_id, "output": "empty"}, + {"type": "function_call_output", "call_id": second_output_call_id, "output": "secret"}, + ] + ) + + assert prompt_for("c2", "c1") == ( + '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' + '{"result_of_call":2}\nempty\n{"result_of_call":1}\nsecret' + ) + assert prompt_for("c1", "c2") != prompt_for("c2", "c1") + + def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): from litellm.caching.redis_semantic_cache import RedisSemanticCache diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 4c1af95f4fe2..2bd593bea0d4 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -7,8 +7,6 @@ import pytest -from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message - from litellm.litellm_core_utils.prompt_templates.common_utils import ( ENCRYPTED_REASONING_SIGNATURE_PREFIX, TOOL_RESULT_IMAGE_BOUNDARY, @@ -32,6 +30,7 @@ system_messages_first, update_messages_with_model_file_ids, ) +from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' @@ -2063,9 +2062,51 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' - '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}ok', + '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1}ok', id="openai-tool-calls-in-order-before-tool-result", ), + pytest.param( + [ + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, + {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, + {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, + ], + }, + ], + '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' + '{"result_of_call":2}B{"result_of_call":1}A', + id="anthropic-parallel-tool-results-tagged-with-their-call", + ), + pytest.param( + [ + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + ], + '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' + '{"result_of_call":2}C', + id="reused-call-ids-keep-first-position", + ), pytest.param( [ Message( @@ -2120,3 +2161,35 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj ) def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures + + +def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": f"{call_id_prefix}1", "type": "function", "function": {"name": "read", "arguments": '"a"'}}, + {"id": f"{call_id_prefix}2", "type": "function", "function": {"name": "read", "arguments": '"b"'}}, + ], + }, + {"role": "tool", "tool_call_id": f"{call_id_prefix}1", "content": result_for_a}, + {"role": "tool", "tool_call_id": f"{call_id_prefix}2", "content": result_for_b}, + ] + + +def _results_in_swapped_order(result_for_a: str, result_for_b: str) -> list[object]: + call, answer_a, answer_b = _parallel_reads(result_for_a, result_for_b) + return [call, answer_b, answer_a] + + +def test_get_str_from_messages_with_tools_tells_apart_parallel_results_answering_different_calls() -> None: + assert get_str_from_messages_with_tools(_parallel_reads("empty", "secret")) != get_str_from_messages_with_tools( + _results_in_swapped_order("secret", "empty") + ) + + +def test_get_str_from_messages_with_tools_ignores_call_ids_that_differ_between_sessions() -> None: + assert get_str_from_messages_with_tools( + _parallel_reads("A", "B", call_id_prefix="toolu_") + ) == get_str_from_messages_with_tools(_parallel_reads("A", "B")) From 6b4b3754d23489e03cc81da0661d5d7d024b83ed Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 18:27:44 +0000 Subject: [PATCH 06/29] fix(caching): encode tool result position and output together so tool text cannot forge result tags Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 56 ++++++++++------ litellm-rust/crates/cache/tests/semantic.rs | 66 ++++++++++++------- litellm/caching/redis_semantic_cache.py | 21 +++--- .../prompt_templates/common_utils.py | 21 +++--- .../unit/caching/test_redis_semantic_cache.py | 29 ++++++-- ...ore_utils_prompt_templates_common_utils.py | 38 ++++++----- 6 files changed, 148 insertions(+), 83 deletions(-) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 990f5e47aa85..253efb477596 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -84,7 +84,7 @@ impl Embedder for PreparedEmbedding { } /// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, -/// then its OpenAI `tool_calls`, then its search results. Each tool result is tagged with the +/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the /// position of the call it answers. pub fn str_from_messages(messages: &[Value]) -> String { let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); @@ -107,12 +107,16 @@ pub fn str_from_messages(messages: &[Value]) -> String { let mut text = String::new(); for message in messages { if message.get("role").and_then(Value::as_str) == Some("tool") { - text.push_str(&tool_result_tag( + let mut output = String::new(); + push_content_text(&mut output, message.get("content"), &call_ordinals); + text.push_str(&tool_result_json( message.get("tool_call_id"), &call_ordinals, + &output, )); + } else { + push_content_text(&mut text, message.get("content"), &call_ordinals); } - push_content_text(&mut text, message.get("content"), &call_ordinals); if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { for tool_call in tool_calls.iter().filter_map(Value::as_object) { let function = tool_call.get("function"); @@ -142,8 +146,13 @@ fn push_content_text( text.push_str(&tool_call_json(block.get("name"), block.get("input"))); } Some("tool_result") => { - text.push_str(&tool_result_tag(block.get("tool_use_id"), call_ordinals)); - push_content_text(text, block.get("content"), call_ordinals); + let mut output = String::new(); + push_content_text(&mut output, block.get("content"), call_ordinals); + text.push_str(&tool_result_json( + block.get("tool_use_id"), + call_ordinals, + &output, + )); } _ => { if let Some(block_text) = block.get("text").and_then(Value::as_str) { @@ -167,14 +176,21 @@ fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap< ordinals } -/// `tool_result_str`: `{"result_of_call":N}` for a result answering a known call, else empty. -fn tool_result_tag(call_id: Option<&Value>, call_ordinals: &HashMap<&str, usize>) -> String { - call_id +/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result +/// answers no known call. +fn tool_result_json( + call_id: Option<&Value>, + call_ordinals: &HashMap<&str, usize>, + output: &str, +) -> String { + let ordinal = call_id .and_then(Value::as_str) - .and_then(|call_id| call_ordinals.get(call_id)) - .map_or_else(String::new, |ordinal| { - format!("{{\"result_of_call\":{ordinal}}}") - }) + .and_then(|call_id| call_ordinals.get(call_id)); + format!( + "{{\"result_of_call\":{},\"output\":{}}}", + compact_json(&json!(ordinal)), + compact_json(&Value::String(output.to_owned())), + ) } /// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. @@ -258,14 +274,16 @@ fn collect_input_text( return; } if map.get("type").and_then(Value::as_str) == Some("function_call_output") { - let tag = tool_result_tag(map.get("call_id"), call_ordinals); - if !tag.is_empty() { - parts.push(tag); - if let Some(output) = map.get("output") { - collect_input_text(output, parts, call_ordinals); - } - return; + let mut output_parts = Vec::new(); + if let Some(output) = map.get("output") { + collect_input_text(output, &mut output_parts, call_ordinals); } + parts.push(tool_result_json( + map.get("call_id"), + call_ordinals, + python_strip(&output_parts.join("\n")), + )); + return; } if let Some(content) = map.get("content").filter(|content| !content.is_null()) { collect_input_text(content, parts, call_ordinals); diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index 7ede4948044d..af87e6e0164a 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -38,67 +38,67 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex json!([{"role": "tool", "content": "small", "search_results": [ {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, ]}]), - "smallsthidden payload", + r#"{"result_of_call":null,"output":"small"}sthidden payload"#, )] #[case::title_only_search_result( json!([{"role": "tool", "content": "small", "search_results": [ {"source": "s", "title": "long title", "content": []}, ]}]), - "smallslong title", + r#"{"result_of_call":null,"output":"small"}slong title"#, )] #[case::search_results_without_content( json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), - "st", + r#"{"result_of_call":null,"output":""}st"#, )] #[case::search_result_fields_in_python_order( json!([{"role": "tool", "content": "c", "search_results": [ {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, {"source": "s2"}, ]}]), - r#"cstbody{"enabled":true}s2"#, + r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, )] #[case::null_citations_skipped( json!([{"role": "tool", "content": "c", "search_results": [ {"source": "s", "citations": null}, ]}]), - "cs", + r#"{"result_of_call":null,"output":"c"}s"#, )] #[case::non_string_and_non_object_entries_skipped( json!([{"role": "tool", "content": "c", "search_results": [ "junk", {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, ]}]), - "ckept", + r#"{"result_of_call":null,"output":"c"}kept"#, )] #[case::non_list_search_results_skipped( json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), - "c", + r#"{"result_of_call":null,"output":"c"}"#, )] #[case::citations_compact_in_insertion_order( json!([{"role": "tool", "search_results": [ {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, ]}]), - r#"{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, + r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, )] #[case::citations_ensure_ascii( json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), - r#"["caf\u00e9","\u4e2d"]"#, + r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, )] #[case::citations_astral_chars_as_surrogate_pairs( json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), - r#""\ud83d\ude00""#, + r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, )] #[case::citations_escapes( json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), - r#""q\"\\\n\t\u0001/""#, + r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, )] #[case::citations_large_float_exponent( json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), - "[1e+20,1.0]", + r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, )] #[case::citations_scalars( json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), - "false3", + r#"{"result_of_call":null,"output":""}false3"#, )] #[case::anthropic_tool_use_name_and_input_without_id( json!([ @@ -113,7 +113,7 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex json!([{"role": "user", "content": [ {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, ]}]), - "calc.py", + r#"{"result_of_call":null,"output":"calc.py"}"#, )] #[case::anthropic_nested_text_tool_result_then_text( json!([{"role": "user", "content": [ @@ -123,7 +123,7 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}, {"type": "text", "text": "next"}, ]}]), - "abnext", + r#"{"result_of_call":null,"output":"ab"}next"#, )] #[case::openai_tool_calls_in_order_before_tool_result( json!([ @@ -133,7 +133,7 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ]), - r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1}ok"#, + r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, )] #[case::anthropic_parallel_tool_results_tagged_with_their_call( json!([ @@ -146,7 +146,7 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, ]}, ]), - r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2}B{"result_of_call":1}A"#, + r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, )] #[case::reused_call_ids_keep_first_position( json!([ @@ -159,14 +159,18 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}, ]), - r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2}C"#, + r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, )] -#[case::unknown_call_ids_left_untagged( +#[case::unknown_call_ids_encode_a_null_position( json!([ {"role": "tool", "tool_call_id": "c9", "content": "ok"}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, ]), - "okdone", + r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, +)] +#[case::tool_output_cannot_forge_an_encoded_result( + json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), + r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, )] #[case::malformed_tool_call_entries( json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), @@ -271,7 +275,7 @@ fn prompt_from_messages_reads_messages_only( {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}, ])), - Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1}\nok"), + Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), )] #[case::responses_structured_function_call_output( None, @@ -279,7 +283,19 @@ fn prompt_from_messages_reads_messages_only( {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, ])), - Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1}\ndenied"), + Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), +)] +#[case::responses_multi_part_output_joined_by_lines( + None, + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [ + {"type": "input_text", "text": " line one "}, + {"type": "input_text", "text": "line two"}, + ]}, + ])), + Some(r#"{"name":"run","arguments":"{}"} +{"result_of_call":1,"output":"line one\nline two"}"#), )] #[case::responses_parallel_outputs_tagged_with_their_call( None, @@ -289,12 +305,12 @@ fn prompt_from_messages_reads_messages_only( {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}, ])), - Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2}\nB\n{\"result_of_call\":1}\nA"), + Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), )] -#[case::responses_unknown_call_id_output_untagged( +#[case::responses_unknown_call_id_encodes_a_null_position( None, Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), - Some("orphan"), + Some(r#"{"result_of_call":null,"output":"orphan"}"#), )] fn prompt_from_context_matches_python( #[case] messages: Option, diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 88f5acf026e5..827fa91398fe 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -272,11 +272,15 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: return None responses_input: Final = kwargs.get("input") - prompt_parts: Final[list[str]] = [] - cls._collect_responses_input_text(responses_input, prompt_parts, cls._responses_call_ordinals(responses_input)) - prompt: Final = "\n".join(prompt_parts).strip() + prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) return prompt or None + @classmethod + def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: + prompt_parts: Final[list[str]] = [] + cls._collect_responses_input_text(value, prompt_parts, call_ordinals) + return "\n".join(prompt_parts).strip() + @classmethod def _collect_responses_input_text( cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] @@ -341,16 +345,17 @@ def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: if isinstance(item, dict) and item.get("type") == "function_call" ) - @staticmethod - def _function_call_as_prompt(value: object, call_ordinals: Mapping[str, int]) -> object: + @classmethod + def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: if not isinstance(value, dict): return value if value.get("type") == "function_call": return tool_call_str(value.get("name"), value.get("arguments")) - result_tag: Final = ( - tool_result_str(value.get("call_id"), call_ordinals) if value.get("type") == "function_call_output" else "" + if value.get("type") != "function_call_output": + return value + return tool_result_str( + value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) ) - return (result_tag, value.get("output")) if result_tag else value @staticmethod def _coerce_response_input_value(value: object) -> object: diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index dddb9f3b4682..0c52fae86d79 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -199,7 +199,7 @@ def get_str_from_messages_with_tools(messages: object) -> str: """ ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) - produce different text. Each result is tagged with the position of the call it answers, since call ids are + produce different text. Each result is encoded with the position of the call it answers, since call ids are random per session """ message_mappings: Final = tuple(_str_mappings(messages)) @@ -211,9 +211,9 @@ def tool_call_str(name: object, arguments: object) -> str: return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' -def tool_result_str(call_id: object, call_ordinals: Mapping[str, int]) -> str: +def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None - return "" if ordinal is None else f'{{"result_of_call":{ordinal}}}' + return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: @@ -234,12 +234,13 @@ def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - result_tag: Final = ( - tool_result_str(message.get("tool_call_id"), call_ordinals) if message.get("role") == "tool" else "" - ) + content: Final = _content_str_with_tools(message.get("content"), call_ordinals) return ( - result_tag - + _content_str_with_tools(message.get("content"), call_ordinals) + ( + tool_result_str(message.get("tool_call_id"), call_ordinals, content) + if message.get("role") == "tool" + else content + ) + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) + extract_search_results_text(message.get("search_results")) ) @@ -256,8 +257,8 @@ def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[st if block_type == "tool_use": return tool_call_str(block.get("name"), block.get("input")) if block_type == "tool_result": - return tool_result_str(block.get("tool_use_id"), call_ordinals) + _content_str_with_tools( - block.get("content"), call_ordinals + return tool_result_str( + block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) ) text: Final = block.get("text") return text if isinstance(text, str) else "" diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 6ca4481648b6..3359796ec84d 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -593,10 +593,10 @@ def turn(command: str) -> list[dict[str, object]]: ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1}ok' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' ) assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1}ok' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' ) @@ -612,7 +612,7 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ) assert prompt == ( - 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1}\nok' + 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' ) @@ -633,10 +633,29 @@ def prompt_for(output_text: str) -> str | None: ) expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' - assert prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1}}\nwrote 5 bytes' + assert ( + prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' + ) assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") +def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], + }, + ] + ) + + assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' + + def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -652,7 +671,7 @@ def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | N assert prompt_for("c2", "c1") == ( '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' - '{"result_of_call":2}\nempty\n{"result_of_call":1}\nsecret' + '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' ) assert prompt_for("c1", "c2") != prompt_for("c2", "c1") diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 2bd593bea0d4..59663de68d05 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -2028,7 +2028,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ), pytest.param( [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - "fix the failing testcalc.py", + 'fix the failing test{"result_of_call":null,"output":"calc.py"}', id="anthropic-string-tool-result", ), pytest.param( @@ -2045,7 +2045,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ], } ], - "abnext", + '{"result_of_call":null,"output":"ab"}next', id="anthropic-nested-text-tool-result-then-text", ), pytest.param( @@ -2062,7 +2062,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' - '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1}ok', + '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', id="openai-tool-calls-in-order-before-tool-result", ), pytest.param( @@ -2083,7 +2083,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s }, ], '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' - '{"result_of_call":2}B{"result_of_call":1}A', + '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', id="anthropic-parallel-tool-results-tagged-with-their-call", ), pytest.param( @@ -2104,9 +2104,26 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s {"role": "tool", "tool_call_id": "call_1", "content": "C"}, ], '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' - '{"result_of_call":2}C', + '{"result_of_call":2,"output":"C"}', id="reused-call-ids-keep-first-position", ), + pytest.param( + [ + { + "role": "tool", + "tool_call_id": "c1", + "content": "small", + "search_results": [{"source": "s", "title": "t"}], + } + ], + '{"result_of_call":null,"output":"small"}st', + id="tool-result-search-results-follow-the-encoded-result", + ), + pytest.param( + [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], + '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', + id="tool-output-cannot-forge-an-encoded-result", + ), pytest.param( [ Message( @@ -2145,17 +2162,6 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj ], id="text-and-image-parts", ), - pytest.param( - [ - { - "role": "tool", - "tool_call_id": "c1", - "content": "small", - "search_results": [{"source": "s", "title": "t"}], - } - ], - id="search-results", - ), pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) From 9379f65201e9e31722223ba6bdf481e6b4b63aa7 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 18:52:07 +0000 Subject: [PATCH 07/29] test(caching): expect encoded tool result record in qdrant semantic prompt parity case Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index fe25c503a368..7a27519c2691 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"sourcetitlebody{"page":1,"section":"intro"}"# + r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( From 127506254b55ca973a9bdb5eb8202f653dfa62b2 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 19:33:17 +0000 Subject: [PATCH 08/29] test(caching): cover tool result arrangements, SDK clients, concurrency and qdrant outage for semantic cache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../caching/test_semantic_cache_tool_turns.py | 265 +++++++++++++++++- 1 file changed, 264 insertions(+), 1 deletion(-) diff --git a/tests/integration/caching/test_semantic_cache_tool_turns.py b/tests/integration/caching/test_semantic_cache_tool_turns.py index 5d6e6a956a16..ac80b806c402 100644 --- a/tests/integration/caching/test_semantic_cache_tool_turns.py +++ b/tests/integration/caching/test_semantic_cache_tool_turns.py @@ -1,14 +1,19 @@ +import asyncio import hashlib import json import math import threading import uuid -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from pathlib import Path from typing import Final from urllib.parse import urlsplit +import anthropic +import httpx +import openai import pytest import yaml from integration._support.client import Gateway, eventually, gateway_from_environment @@ -46,6 +51,8 @@ class _Peer: points: list[Mapping[str, JsonValue]] = field(default_factory=list) # mutable-ok: the Qdrant collection embedded: list[str] = field(default_factory=list) # mutable-ok: every prompt the proxy embedded answered: list[str] = field(default_factory=list) # mutable-ok: every completion the provider served + qdrant_down: threading.Event = field(default_factory=threading.Event) + refused_writes: list[str] = field(default_factory=list) # mutable-ok: upserts refused while Qdrant is down def stored(self) -> int: with self.lock: @@ -55,6 +62,11 @@ def respond(self, request: Request) -> Reply: path: Final = urlsplit(request.target).path body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} collection: Final = f"/qdrant/collections/{_COLLECTION}" + if path.startswith("/qdrant/") and self.qdrant_down.is_set(): + if path == f"{collection}/points": + with self.lock: + self.refused_writes.append(path) + return Reply(status=503, body=b'{"status":{"error":"qdrant unavailable"}}') if request.method == "GET" and path == f"{collection}/exists": return self._json({"result": {"exists": False}, "status": "ok"}) if request.method in {"GET", "PUT"} and path in {collection, f"{collection}/index"}: @@ -279,3 +291,254 @@ def wrote(path: str) -> list[JsonValue]: assert len(set(answers[:2])) == 2, f"a different tool call replayed an earlier cached answer: {answers}" assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered assert answers[2] == answers[0], f"a repeated tool call missed the cache: {answers}" + + +def _tool_turn(task: str, calls: Sequence[tuple[str, str]], results: Sequence[tuple[str, str]]) -> list[JsonValue]: + return [ + {"role": "user", "content": task}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": "read_file", "arguments": arguments}} + for call_id, arguments in calls + ], + }, + *({"role": "tool", "tool_call_id": call_id, "content": output} for call_id, output in results), + ] + + +def _claude_turn(task: str, calls: Sequence[tuple[str, str]], results: Sequence[tuple[str, str]]) -> list[JsonValue]: + return [ + {"role": "user", "content": task}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": call_id, "name": "Read", "input": {"file_path": file_path}} + for call_id, file_path in calls + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": call_id, "content": output} for call_id, output in results + ], + }, + ] + + +_TWO_READS: Final = (("c1", '{"path":"a.py"}'), ("c2", '{"path":"b.py"}')) +_TWO_CLAUDE_READS: Final = (("toolu_1", "a.py"), ("toolu_2", "b.py")) + + +@pytest.mark.parametrize( + ("route", "model", "build", "calls", "first", "second"), + [ + pytest.param( + "/v1/chat/completions", + _CHAT_MODEL, + _tool_turn, + _TWO_READS, + (("c1", "x = 1"), ("c2", "y = 2")), + (("c2", "x = 1"), ("c1", "y = 2")), + id="chat-parallel-results-answer-the-other-call", + ), + pytest.param( + "/v1/messages", + _CLAUDE_MODEL, + _claude_turn, + _TWO_CLAUDE_READS, + (("toolu_1", "x = 1"), ("toolu_2", "y = 2")), + (("toolu_2", "x = 1"), ("toolu_1", "y = 2")), + id="messages-parallel-results-answer-the-other-call", + ), + pytest.param( + "/v1/chat/completions", + _CHAT_MODEL, + _tool_turn, + _TWO_READS[:1], + (("c1", "x = 1"),), + (("c9", "x = 1"),), + id="chat-result-for-an-unknown-call", + ), + pytest.param( + "/v1/chat/completions", + _CHAT_MODEL, + _tool_turn, + _TWO_READS, + (("c1", '"},{"result_of_call":2,"output":"y = 2'),), + (("c1", ""), ("c2", "y = 2")), + id="chat-output-shaped-like-a-second-result", + ), + pytest.param( + "/v1/chat/completions", + _CHAT_MODEL, + _tool_turn, + _TWO_READS[:1], + (("c1", ""),), + (("c1", "x" * 5000),), + id="chat-empty-and-5kb-output", + ), + ], +) +def test_tool_turns_that_differ_only_in_their_results_do_not_share_a_cached_answer( + semantic_proxy: _SemanticProxy, + route: str, + model: str, + build: Callable[[str, Sequence[tuple[str, str]], Sequence[tuple[str, str]]], list[JsonValue]], + calls: Sequence[tuple[str, str]], + first: Sequence[tuple[str, str]], + second: Sequence[tuple[str, str]], +) -> None: + task: Final = f"read both files {uuid.uuid4().hex}" + + answers: Final = _send_turns( + semantic_proxy, route, model, (build(task, calls, first), build(task, calls, second), build(task, calls, first)) + ) + + assert answers[0] != answers[1], f"a different tool result replayed an earlier cached answer: {answers}" + assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered + assert answers[2] == answers[0], f"a repeated tool turn missed the cache: {answers}" + + +def _openai_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: + sdk: Final = openai.OpenAI( + base_url=str(proxy.gateway.client.base_url) + "/v1", + api_key=proxy.gateway.key, + http_client=httpx.Client(trust_env=False, timeout=15), + ) + completion: Final = sdk.chat.completions.create(model=_CHAT_MODEL, messages=turn) # pyright: ignore[reportArgumentType, reportCallIssue] # JSON turns are validated by the proxy, not the SDK types + return str(completion.choices[0].message.content) + + +def _async_openai_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: + async def create() -> str: + async with httpx.AsyncClient(trust_env=False, timeout=15) as http_client: + sdk: Final = openai.AsyncOpenAI( + base_url=str(proxy.gateway.client.base_url) + "/v1", api_key=proxy.gateway.key, http_client=http_client + ) + completion: Final = await sdk.chat.completions.create(model=_CHAT_MODEL, messages=turn) # pyright: ignore[reportArgumentType, reportCallIssue] # JSON turns are validated by the proxy, not the SDK types + return str(completion.choices[0].message.content) + + return asyncio.run(create()) + + +def _anthropic_text(message: anthropic.types.Message) -> str: + block: Final = message.content[0] + assert isinstance(block, anthropic.types.TextBlock), message + return block.text + + +def _anthropic_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: + sdk: Final = anthropic.Anthropic( + base_url=str(proxy.gateway.client.base_url), + api_key=proxy.gateway.key, + http_client=httpx.Client(trust_env=False, timeout=15), + ) + return _anthropic_text(sdk.messages.create(model=_CLAUDE_MODEL, max_tokens=16, messages=turn)) # pyright: ignore[reportArgumentType] # JSON turns are validated by the proxy, not the SDK types + + +def _async_anthropic_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: + async def create() -> str: + async with httpx.AsyncClient(trust_env=False, timeout=15) as http_client: + sdk: Final = anthropic.AsyncAnthropic( + base_url=str(proxy.gateway.client.base_url), api_key=proxy.gateway.key, http_client=http_client + ) + message: Final = await sdk.messages.create(model=_CLAUDE_MODEL, max_tokens=16, messages=turn) # pyright: ignore[reportArgumentType] # JSON turns are validated by the proxy, not the SDK types + return _anthropic_text(message) + + return asyncio.run(create()) + + +@pytest.mark.parametrize( + ("send", "build"), + [ + pytest.param(_openai_sdk, _tool_turn, id="openai-sdk"), + pytest.param(_async_openai_sdk, _tool_turn, id="async-openai-sdk"), + pytest.param(_anthropic_sdk, _claude_turn, id="anthropic-sdk"), + pytest.param(_async_anthropic_sdk, _claude_turn, id="async-anthropic-sdk"), + ], +) +def test_sdk_clients_get_a_fresh_answer_for_each_tool_turn_and_a_hit_for_a_repeat( + semantic_proxy: _SemanticProxy, + send: Callable[[_SemanticProxy, list[JsonValue]], str], + build: Callable[[str, Sequence[tuple[str, str]], Sequence[tuple[str, str]]], list[JsonValue]], +) -> None: + task: Final = f"read the file {uuid.uuid4().hex}" + calls: Final = _TWO_READS[:1] if build is _tool_turn else _TWO_CLAUDE_READS[:1] + call_id: Final = calls[0][0] + alpha: Final = build(task, calls, ((call_id, "x = 1"),)) + beta: Final = build(task, calls, ((call_id, "PermissionError"),)) + + def answer(turn: list[JsonValue]) -> str: + stored_before: Final = semantic_proxy.peer.stored() + answered_before: Final = len(semantic_proxy.peer.answered) + text: Final = send(semantic_proxy, turn) + if len(semantic_proxy.peer.answered) > answered_before: + eventually(semantic_proxy.peer.stored, lambda count: count > stored_before) + return text + + answers: Final = [answer(turn) for turn in (alpha, beta, alpha)] + + assert answers[0] != answers[1], f"a different tool result replayed an earlier cached answer: {answers}" + assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered + assert answers[2] == answers[0], f"a repeated tool turn missed the cache: {answers}" + + +def test_concurrent_agent_loops_each_get_their_own_answer_and_their_own_hit(semantic_proxy: _SemanticProxy) -> None: + task: Final = f"read the file {uuid.uuid4().hex}" + turns: Final = tuple(_tool_turn(task, _TWO_READS[:1], (("c1", f"line {index}"),)) for index in range(8)) + + def send(turn: list[JsonValue]) -> str: + response: Final = semantic_proxy.gateway.request( + "POST", "/v1/chat/completions", {"model": _CHAT_MODEL, "messages": turn} + ) + assert response.status_code == 200, response.text + choice: Final = _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_json(response.content)["choices"])[0]) + return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) + + stored_before: Final = semantic_proxy.peer.stored() + with ThreadPoolExecutor(max_workers=len(turns)) as pool: + first: Final = tuple(pool.map(send, turns)) + eventually(semantic_proxy.peer.stored, lambda count: count >= stored_before + len(turns)) + answered_before: Final = len(semantic_proxy.peer.answered) + with ThreadPoolExecutor(max_workers=len(turns)) as pool: + second: Final = tuple(pool.map(send, turns)) + + assert len(set(first)) == len(turns), f"concurrent tool turns shared a cached answer: {first}" + assert second == first, f"a repeated tool turn got another turn's answer: {first} {second}" + assert len(semantic_proxy.peer.answered) == answered_before, "a repeated tool turn missed the cache" + + +def test_tool_turns_are_answered_while_qdrant_is_down_and_cached_once_it_recovers( + semantic_proxy: _SemanticProxy, +) -> None: + task: Final = f"read the file {uuid.uuid4().hex}" + turn: Final = _tool_turn(task, _TWO_READS[:1], (("c1", "x = 1"),)) + body: Final[dict[str, JsonValue]] = {"model": _CHAT_MODEL, "messages": turn} + + def send() -> httpx.Response: + return semantic_proxy.gateway.request("POST", "/v1/chat/completions", body) + + refused_before: Final = len(semantic_proxy.peer.refused_writes) + answered_before: Final = len(semantic_proxy.peer.answered) + semantic_proxy.peer.qdrant_down.set() + try: + outage: Final = (send(), send()) + eventually(lambda: len(semantic_proxy.peer.refused_writes), lambda count: count >= refused_before + 2) + finally: + semantic_proxy.peer.qdrant_down.clear() + answered_during_outage: Final = len(semantic_proxy.peer.answered) - answered_before + stored_before: Final = semantic_proxy.peer.stored() + recovered: Final = send() + eventually(semantic_proxy.peer.stored, lambda count: count > stored_before) + answered_after_recovery: Final = len(semantic_proxy.peer.answered) + repeat: Final = send() + + assert [response.status_code for response in outage] == [200, 200], [response.text for response in outage] + assert answered_during_outage == 2, "an outage request was not sent to the provider" + assert recovered.status_code == 200, recovered.text + assert answered_after_recovery - answered_before == 3, "the first request after recovery hit a stale entry" + assert repeat.status_code == 200, repeat.text + assert repeat.json()["choices"] == recovered.json()["choices"] + assert len(semantic_proxy.peer.answered) == answered_after_recovery, "the repeat after recovery missed the cache" From 5aed3ab3de3597dec22d3afcb33db7196f0a2a24 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:09:15 +0000 Subject: [PATCH 09/29] feat(caching): embed every semantic cache prompt field except volatile ones Replace the per-shape allowlist in the Python and Rust semantic cache prompt walkers with one include-by-default walker. Plain text keeps its old concatenation; any other block or message is embedded as compact JSON with call ids mapped to ordinals, cache_control dropped, and signatures, encrypted content and base64 data replaced with a short sha256 digest. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/Cargo.lock | 1 + .../cache-qdrant-semantic/tests/qdrant.rs | 8 +- litellm-rust/crates/cache/Cargo.toml | 1 + litellm-rust/crates/cache/src/semantic.rs | 386 ++++++++-------- litellm-rust/crates/cache/tests/semantic.rs | 435 +++++++----------- litellm/caching/redis_semantic_cache.py | 100 +--- .../prompt_templates/common_utils.py | 174 ++++--- .../unit/caching/test_redis_semantic_cache.py | 120 ++--- ...ore_utils_prompt_templates_common_utils.py | 126 +++-- 9 files changed, 609 insertions(+), 742 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 8d189c8c515a..63b4c9352086 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3426,6 +3426,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.19", "tokio", ] diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index 7a27519c2691..2d94d7661463 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -225,7 +225,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { } #[rstest] -#[case::content_parts_skip_images( +#[case::content_parts_keep_images_in_place( json!([ {"role": "user", "content": "hello"}, { @@ -237,9 +237,9 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { ], }, ]), - "helloworld!" + r#"helloworld{"type":"image_url","image_url":{"url":"ignored"}}!"# )] -#[case::search_results_and_compact_citations( +#[case::tool_message_with_search_results_kept_as_json( json!([{ "role": "tool", "content": null, @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# + r#"{"role":"tool","search_results":[{"source":"source","title":"title","content":[{"text":"body"}],"citations":{"page":1,"section":"intro"}}]}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index f18dbd9cb260..69b898adb11c 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } +sha2.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 253efb477596..db14abee664f 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -8,9 +8,10 @@ use std::{collections::HashMap, future::Future, io}; use serde::Serialize; use serde_json::{ - Value, json, + Map, Value, ser::{CharEscape, Formatter, Serializer}, }; +use sha2::{Digest, Sha256}; use crate::{BaseCache, Error, SemanticCacheContext}; @@ -83,235 +84,242 @@ impl Embedder for PreparedEmbedding { } } -/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, -/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the -/// position of the call it answers. +const PLAIN_TYPES: [&str; 4] = ["text", "input_text", "output_text", "message"]; +const PLAIN_KEYS: [&str; 5] = ["role", "type", "text", "content", "status"]; +const TOOL_ROLES: [&str; 2] = ["tool", "function"]; +const IGNORED_KEYS: [&str; 1] = ["cache_control"]; +const CALL_ID_KEYS: [&str; 4] = ["id", "call_id", "tool_use_id", "tool_call_id"]; +const OPAQUE_KEYS: [&str; 4] = ["data", "file_data", "signature", "encrypted_content"]; + +/// `get_str_from_messages_with_tools`: plain text parts stay plain text, and every other part is +/// kept as compact JSON with call ids replaced by their position and opaque blobs by a digest. pub fn str_from_messages(messages: &[Value]) -> String { - let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); - let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { - let block_ids = message - .get("content") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) - .filter_map(|block| block.get("id")); - let tool_call_ids = message - .get("tool_calls") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|tool_call| tool_call.get("id")); - block_ids.chain(tool_call_ids) - })); - let mut text = String::new(); - for message in messages { - if message.get("role").and_then(Value::as_str) == Some("tool") { - let mut output = String::new(); - push_content_text(&mut output, message.get("content"), &call_ordinals); - text.push_str(&tool_result_json( - message.get("tool_call_id"), - &call_ordinals, - &output, - )); - } else { - push_content_text(&mut text, message.get("content"), &call_ordinals); - } - if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { - for tool_call in tool_calls.iter().filter_map(Value::as_object) { - let function = tool_call.get("function"); - text.push_str(&tool_call_json( - function.and_then(|function| function.get("name")), - function.and_then(|function| function.get("arguments")), - )); - } - } - push_search_results_text(&mut text, message.get("search_results")); + let messages: Vec<&Value> = messages + .iter() + .filter(|message| message.is_object()) + .collect(); + let call_ordinals = call_id_ordinals(messages.iter().copied()); + messages + .into_iter() + .filter_map(Value::as_object) + .map(|message| message_prompt(message, &call_ordinals)) + .collect() +} + +/// `get_str_from_responses_input`: `str_from_messages` for a Responses API `input`, one stripped +/// line per part. `None` when nothing is left. +fn str_from_responses_input(input: &Value) -> Option { + let call_ordinals = call_id_ordinals(std::iter::once(input)); + let mut parts = Vec::new(); + push_responses_input_parts(input, &call_ordinals, &mut parts); + let prompt = python_strip(&parts.join("\n")).to_owned(); + (!prompt.is_empty()).then_some(prompt) +} + +fn message_prompt(message: &Map, call_ordinals: &HashMap<&str, usize>) -> String { + let is_tool = message + .get("role") + .and_then(Value::as_str) + .is_some_and(|role| TOOL_ROLES.contains(&role)); + if is_tool || !is_plain(message) { + return compact_json(&normalized(message, call_ordinals)); } - text + plain_prompt(message, call_ordinals) } -/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. -fn push_content_text( - text: &mut String, +fn plain_prompt(value: &Map, call_ordinals: &HashMap<&str, usize>) -> String { + let mut prompt = value + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + push_content_prompt(&mut prompt, value.get("content"), call_ordinals); + prompt +} + +fn push_content_prompt( + prompt: &mut String, content: Option<&Value>, call_ordinals: &HashMap<&str, usize>, ) { match content { - Some(Value::String(content)) => text.push_str(content), - Some(Value::Array(blocks)) => { - for block in blocks.iter().filter_map(Value::as_object) { - match block.get("type").and_then(Value::as_str) { - Some("tool_use") => { - text.push_str(&tool_call_json(block.get("name"), block.get("input"))); - } - Some("tool_result") => { - let mut output = String::new(); - push_content_text(&mut output, block.get("content"), call_ordinals); - text.push_str(&tool_result_json( - block.get("tool_use_id"), - call_ordinals, - &output, - )); - } - _ => { - if let Some(block_text) = block.get("text").and_then(Value::as_str) { - text.push_str(block_text); - } - } - } + None | Some(Value::Null) => {} + Some(Value::String(text)) => prompt.push_str(text), + Some(Value::Array(items)) => { + for item in items { + push_content_prompt(prompt, Some(item), call_ordinals); } } - _ => {} + Some(Value::Object(map)) if is_plain(map) => { + prompt.push_str(&plain_prompt(map, call_ordinals)); + } + Some(Value::Object(map)) => prompt.push_str(&compact_json(&normalized(map, call_ordinals))), + Some(other) => prompt.push_str(&compact_json(other)), } } -/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. -fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { - let mut ordinals = HashMap::new(); - for call_id in call_ids.filter_map(Value::as_str) { - let next = ordinals.len() + 1; - ordinals.entry(call_id).or_insert(next); +fn push_responses_input_parts( + value: &Value, + call_ordinals: &HashMap<&str, usize>, + parts: &mut Vec, +) { + match value { + Value::Null => {} + Value::String(text) => push_stripped(text, parts), + Value::Array(items) => { + for item in items { + push_responses_input_parts(item, call_ordinals, parts); + } + } + Value::Object(map) if is_plain(map) => { + push_stripped( + map.get("text").and_then(Value::as_str).unwrap_or_default(), + parts, + ); + if let Some(content) = map.get("content") { + push_responses_input_parts(content, call_ordinals, parts); + } + } + Value::Object(map) => parts.push(compact_json(&normalized(map, call_ordinals))), + other => parts.push(compact_json(other)), } - ordinals } -/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result -/// answers no known call. -fn tool_result_json( - call_id: Option<&Value>, - call_ordinals: &HashMap<&str, usize>, - output: &str, -) -> String { - let ordinal = call_id - .and_then(Value::as_str) - .and_then(|call_id| call_ordinals.get(call_id)); - format!( - "{{\"result_of_call\":{},\"output\":{}}}", - compact_json(&json!(ordinal)), - compact_json(&Value::String(output.to_owned())), - ) +fn push_stripped(text: &str, parts: &mut Vec) { + let stripped = python_strip(text); + if !stripped.is_empty() { + parts.push(stripped.to_owned()); + } } -/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. -fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { - compact_json(&json!({ - "name": name.unwrap_or(&Value::Null), - "arguments": arguments.unwrap_or(&Value::Null), - })) +/// `_is_plain`: a text-only part whose every kept key is a plain or call id key. +fn is_plain(value: &Map) -> bool { + let plain_type = match value.get("type") { + None | Some(Value::Null) => true, + Some(Value::String(value_type)) => PLAIN_TYPES.contains(&value_type.as_str()), + Some(_) => false, + }; + let plain_text = matches!( + value.get("text"), + None | Some(Value::Null | Value::String(_)) + ); + plain_type + && plain_text + && kept_entries(value).all(|(key, _)| { + PLAIN_KEYS.contains(&key.as_str()) || CALL_ID_KEYS.contains(&key.as_str()) + }) } -/// The messages prompt Qdrant embeds: `None` when the request carries no messages. -pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { - let messages = context.messages.as_ref()?.as_array()?; - (!messages.is_empty()).then(|| str_from_messages(messages)) +fn kept_entries(value: &Map) -> impl Iterator { + value + .iter() + .filter(|(key, item)| !IGNORED_KEYS.contains(&key.as_str()) && !is_empty(item)) } -/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a -/// Responses API `input`. `None` when neither yields a prompt. -pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { - if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) - && !messages.is_empty() - { - return Some(str_from_messages(messages)); +fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(text) => text.is_empty(), + Value::Array(items) => items.is_empty(), + Value::Object(map) => map.is_empty(), + Value::Bool(_) | Value::Number(_) => false, } - let input = context.input.as_ref()?; - let call_ordinals = tool_call_ordinals( - input - .as_array() - .into_iter() - .flatten() - .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) - .filter_map(|item| item.get("call_id")), - ); - let mut parts = Vec::new(); - collect_input_text(input, &mut parts, &call_ordinals); - let prompt = python_strip(&parts.join("\n")).to_owned(); - (!prompt.is_empty()).then_some(prompt) } -/// `extract_search_results_text`. -fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { - let Some(Value::Array(results)) = search_results else { - return; - }; - for result in results.iter().filter_map(Value::as_object) { - for key in ["source", "title"] { - if let Some(value) = result.get(key).and_then(Value::as_str) { - text.push_str(value); - } - } - if let Some(Value::Array(content)) = result.get("content") { - for block in content.iter().filter_map(Value::as_object) { - if let Some(value) = block.get("text").and_then(Value::as_str) { - text.push_str(value); - } - } - } - if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) { - text.push_str(&compact_json(citations)); - } - } +/// `_normalized`: the part minus ignored and empty entries, with call ids replaced by their +/// position and opaque blobs by `sha256:` plus the first 16 hex digits of their digest. +fn normalized(value: &Map, call_ordinals: &HashMap<&str, usize>) -> Value { + Value::Object( + kept_entries(value) + .map(|(key, item)| { + ( + key.clone(), + normalized_value(item, call_ordinals, Some(key)), + ) + }) + .collect(), + ) } -fn collect_input_text( +fn normalized_value( value: &Value, - parts: &mut Vec, call_ordinals: &HashMap<&str, usize>, -) { + key: Option<&str>, +) -> Value { match value { - Value::String(text) => { - push_trimmed(text, parts); - } - Value::Array(items) => { - for item in items { - collect_input_text(item, parts, call_ordinals); - } + Value::Object(map) => normalized(map, call_ordinals), + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| normalized_value(item, call_ordinals, None)) + .collect(), + ), + Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => call_ordinals + .get(text.as_str()) + .map_or_else(|| value.clone(), |ordinal| Value::from(*ordinal)), + Value::String(text) + if key.is_some_and(|key| OPAQUE_KEYS.contains(&key)) || text.starts_with("data:") => + { + Value::String(digest(text)) } + other => other.clone(), + } +} + +fn digest(text: &str) -> String { + let hash = Sha256::digest(text.as_bytes()); + let hex: String = hash[..8].iter().map(|byte| format!("{byte:02x}")).collect(); + format!("sha256:{hex}") +} + +/// `_call_id_ordinals`: the 1-based position of each distinct call id string, first seen first. +fn call_id_ordinals<'a>(values: impl Iterator) -> HashMap<&'a str, usize> { + let mut ids = Vec::new(); + for value in values { + collect_call_ids(value, None, &mut ids); + } + let mut ordinals = HashMap::new(); + for id in ids { + let next = ordinals.len() + 1; + ordinals.entry(id).or_insert(next); + } + ordinals +} + +fn collect_call_ids<'a>(value: &'a Value, key: Option<&str>, ids: &mut Vec<&'a str>) { + match value { + Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => ids.push(text), Value::Object(map) => { - if map.get("type").and_then(Value::as_str) == Some("function_call") { - parts.push(tool_call_json(map.get("name"), map.get("arguments"))); - return; - } - if map.get("type").and_then(Value::as_str) == Some("function_call_output") { - let mut output_parts = Vec::new(); - if let Some(output) = map.get("output") { - collect_input_text(output, &mut output_parts, call_ordinals); + for (child_key, child) in map { + if !IGNORED_KEYS.contains(&child_key.as_str()) { + collect_call_ids(child, Some(child_key), ids); } - parts.push(tool_result_json( - map.get("call_id"), - call_ordinals, - python_strip(&output_parts.join("\n")), - )); - return; } - if let Some(content) = map.get("content").filter(|content| !content.is_null()) { - collect_input_text(content, parts, call_ordinals); - return; - } - for key in ["text", "output", "input_text", "output_text"] { - match map.get(key) { - Some(nested @ Value::Array(_)) => { - collect_input_text(nested, parts, call_ordinals); - return; - } - Some(Value::String(text)) if push_trimmed(text, parts) => return, - _ => {} - } + } + Value::Array(items) => { + for item in items { + collect_call_ids(item, None, ids); } } _ => {} } } -/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left. -fn push_trimmed(text: &str, parts: &mut Vec) -> bool { - let trimmed = python_strip(text); - if trimmed.is_empty() { - return false; +/// The messages prompt Qdrant embeds: `None` when the request carries no messages. +pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { + let messages = context.messages.as_ref()?.as_array()?; + (!messages.is_empty()).then(|| str_from_messages(messages)) +} + +/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a +/// Responses API `input`. `None` when neither yields a prompt. +pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { + if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) + && !messages.is_empty() + { + return Some(str_from_messages(messages)); } - parts.push(trimmed.to_owned()); - true + str_from_responses_input(context.input.as_ref()?) } /// `str.strip()`: Python's whitespace also covers the ASCII information separators. diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index af87e6e0164a..c4a1bfb2d567 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -16,168 +16,115 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex } #[rstest] -#[case::empty(json!([]), "")] -#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")] +#[case::empty(r#"[]"#, r#""#)] +#[case::string_content(r#"[{"role": "user", "content": "hello"}]"#, r#"hello"#)] #[case::concatenates_messages( - json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]), - "be brief. hello", + r#"[{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]"#, + r#"be brief. hello"# )] #[case::text_parts( - json!([{"role": "user", "content": [ - {"type": "text", "text": "What is "}, - {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, - {"type": "text", "text": "this?"}, - ]}]), - "What is this?", + r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "text", "text": "this?"}]}]"#, + r#"What is this?"#, +)] +#[case::image_part_kept_in_place( + r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "this?"}]}]"#, + r#"What is {"type":"image_url","image_url":{"url":"https://example.com/a.png"}}this?"#, )] #[case::missing_null_and_empty_content( - json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), - "", -)] -#[case::search_results_hidden_behind_small_content( - json!([{"role": "tool", "content": "small", "search_results": [ - {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, - ]}]), - r#"{"result_of_call":null,"output":"small"}sthidden payload"#, -)] -#[case::title_only_search_result( - json!([{"role": "tool", "content": "small", "search_results": [ - {"source": "s", "title": "long title", "content": []}, - ]}]), - r#"{"result_of_call":null,"output":"small"}slong title"#, -)] -#[case::search_results_without_content( - json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), - r#"{"result_of_call":null,"output":""}st"#, -)] -#[case::search_result_fields_in_python_order( - json!([{"role": "tool", "content": "c", "search_results": [ - {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, - {"source": "s2"}, - ]}]), - r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, -)] -#[case::null_citations_skipped( - json!([{"role": "tool", "content": "c", "search_results": [ - {"source": "s", "citations": null}, - ]}]), - r#"{"result_of_call":null,"output":"c"}s"#, -)] -#[case::non_string_and_non_object_entries_skipped( - json!([{"role": "tool", "content": "c", "search_results": [ - "junk", - {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, - ]}]), - r#"{"result_of_call":null,"output":"c"}kept"#, -)] -#[case::non_list_search_results_skipped( - json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), - r#"{"result_of_call":null,"output":"c"}"#, -)] -#[case::citations_compact_in_insertion_order( - json!([{"role": "tool", "search_results": [ - {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, - ]}]), - r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, -)] -#[case::citations_ensure_ascii( - json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), - r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, -)] -#[case::citations_astral_chars_as_surrogate_pairs( - json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), - r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, -)] -#[case::citations_escapes( - json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), - r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, -)] -#[case::citations_large_float_exponent( - json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), - r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, -)] -#[case::citations_scalars( - json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), - r#"{"result_of_call":null,"output":""}false3"#, -)] -#[case::anthropic_tool_use_name_and_input_without_id( - json!([ - {"role": "user", "content": "fix the failing test"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, - ]}, - ]), - r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, + r#"[{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]"#, + r#""#, +)] +#[case::non_object_messages_skipped( + r#"["junk", 3, {"role": "user", "content": "kept"}]"#, + r#"kept"# +)] +#[case::cache_control_ignored( + r#"[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}]"#, + r#"hi"#, +)] +#[case::extra_message_field_kept( + r#"[{"role": "user", "name": "alice", "content": "hi"}]"#, + r#"{"role":"user","name":"alice","content":"hi"}"# +)] +#[case::unknown_block_type_kept( + r#"[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}]"#, + r#"{"type":"brand_new_block","payload":{"k":"v"}}!"#, +)] +#[case::non_string_text_kept( + r#"[{"role": "user", "content": [{"type": "text", "text": 3}]}]"#, + r#"{"type":"text","text":3}"# +)] +#[case::scalar_content_kept( + r#"[{"role": "user", "content": 7}, {"role": "user", "content": [1.5, true, "x"]}]"#, + r#"71.5truex"# +)] +#[case::search_results_kept( + r#"[{"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": [{"type": "text", "text": "hidden payload"}], "citations": {"enabled": true}}]}]"#, + r#"{"role":"tool","content":"small","search_results":[{"source":"s","title":"t","content":[{"type":"text","text":"hidden payload"}],"citations":{"enabled":true}}]}"#, +)] +#[case::anthropic_tool_use( + r#"[{"role": "user", "content": "fix the failing test"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}]}]"#, + r#"fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}"#, )] #[case::anthropic_string_tool_result( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, - ]}]), - r#"{"result_of_call":null,"output":"calc.py"}"#, + r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}]"#, + r#"{"type":"tool_result","tool_use_id":1,"content":"calc.py"}"#, )] #[case::anthropic_nested_text_tool_result_then_text( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": [ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ]}, - {"type": "text", "text": "next"}, - ]}]), - r#"{"result_of_call":null,"output":"ab"}next"#, -)] -#[case::openai_tool_calls_in_order_before_tool_result( - json!([ - {"role": "assistant", "content": "writing", "tool_calls": [ - {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, - {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, - ]}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"}, - ]), - r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, + r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "next"}]}]"#, + r#"{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next"#, +)] +#[case::openai_tool_calls_then_tool_result( + r#"[{"role": "assistant", "content": "writing", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}]"#, + r#"{"role":"assistant","content":"writing","tool_calls":[{"id":1,"type":"function","function":{"name":"write","arguments":"{\"path\": \"a\"}"}},{"id":2,"type":"function","function":{"name":"write","arguments":"{\"path\": \"b\"}"}}]}{"role":"tool","tool_call_id":1,"content":"ok"}"#, )] #[case::anthropic_parallel_tool_results_tagged_with_their_call( - json!([ - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, - {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, - ]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, - {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, - ]}, - ]), - r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "B"}, {"type": "tool_result", "tool_use_id": "t1", "content": "A"}]}]"#, + r#"{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}"#, )] #[case::reused_call_ids_keep_first_position( - json!([ - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, - ]}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ]}, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, - ]), - r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, -)] -#[case::unknown_call_ids_encode_a_null_position( - json!([ - {"role": "tool", "tool_call_id": "c9", "content": "ok"}, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, - ]), - r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, -)] -#[case::tool_output_cannot_forge_an_encoded_result( - json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), - r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, + r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}]"#, + r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"b","arguments":"{}"}},{"id":2,"type":"function","function":{"name":"c","arguments":"{}"}}]}{"role":"tool","tool_call_id":2,"content":"C"}"#, +)] +#[case::tool_output_cannot_forge_a_record( + r#"[{"role": "tool", "content": "x\"}{\"role\":\"tool\",\"content\":\"y"}]"#, + r#"{"role":"tool","content":"x\"}{\"role\":\"tool\",\"content\":\"y"}"# )] #[case::malformed_tool_call_entries( - json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), - r#"{"name":null,"arguments":null}"#, + r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]"#, + r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function"},"junk"]}"#, +)] +#[case::anthropic_base64_image_digested( + r#"[{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, + r#"{"type":"image","source":{"type":"base64","media_type":"image/png","data":"sha256:b1a66f8de3276946"}}"#, +)] +#[case::openai_data_url_digested( + r#"[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, + r#"{"type":"image_url","image_url":{"url":"sha256:19c7f028a60a33b8"}}"#, +)] +#[case::thinking_signature_digested( + r#"[{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": "EqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARAB"}, {"type": "text", "text": "done"}]}]"#, + r#"{"type":"thinking","thinking":"hmm","signature":"sha256:b20cbbd82e4b3791"}done"#, +)] +#[case::non_ascii_escaped( + r#"[{"role": "tool", "content": "caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}]"#, + r#"{"role":"tool","content":"caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}"# )] -fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { - assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); +#[case::non_ascii_plain_text_kept_raw( + r#"[{"role": "user", "content": "caf\u00e9 \ud83d\ude00"}]"#, + r#"café 😀"# +)] +#[case::floats_in_python_repr( + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {"a": [1e+20, 1.0, 1.5e-07, 0.0001, 1e+16, -2.5, 3]}}]}]"#, + r#"{"type":"tool_use","id":1,"name":"n","input":{"a":[1e+20,1.0,1.5e-07,0.0001,1e+16,-2.5,3]}}"#, +)] +#[case::empty_fields_dropped( + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {}, "extra": [], "note": ""}]}]"#, + r#"{"type":"tool_use","id":1,"name":"n"}"#, +)] +fn str_from_messages_matches_python(#[case] messages: &str, #[case] expected: &str) { + let messages: Vec = serde_json::from_str(messages).unwrap(); + assert_eq!(str_from_messages(&messages), expected); } #[rstest] @@ -196,167 +143,109 @@ fn prompt_from_messages_reads_messages_only( #[rstest] #[case::prefers_messages( - Some(json!([{"content": "message prompt"}])), - Some(json!("responses prompt")), - Some("message prompt"), + Some(r#"[{"content": "message prompt"}]"#), + Some(r#""responses prompt""#), + Some(r#"message prompt"#) )] #[case::empty_messages_fall_back_to_input( - Some(json!([])), - Some(json!("responses prompt")), - Some("responses prompt"), + Some(r#"[]"#), + Some(r#""responses prompt""#), + Some(r#"responses prompt"#) )] #[case::messages_without_text_keep_an_empty_prompt( - Some(json!([{"content": null}])), - Some(json!("x")), - Some(""), + Some(r#"[{"content": null}]"#), + Some(r#""x""#), + Some(r#""#) )] -#[case::nothing(None, None, None)] -#[case::null_input(None, Some(Value::Null), None)] -#[case::blank_string(None, Some(json!(" ")), None)] +#[case::null_input(None, None, None)] +#[case::blank_string(None, Some(r#"" ""#), None)] #[case::trimmed_string( None, - Some(json!(" What is the capital of France?\n")), - Some("What is the capital of France?"), + Some(r#"" What is the capital of France?\n""#), + Some(r#"What is the capital of France?"#) )] -#[case::image_only( - None, - Some(json!([{"type": "input_image", "image_url": "https://example.com"}])), +#[case::strip_information_separators(None, Some(r#""\u001ca\u001f""#), Some(r#"a"#))] +#[case::image_only_kept( None, + Some(r#"[{"type": "input_image", "image_url": "https://example.com"}]"#), + Some(r#"{"type":"input_image","image_url":"https://example.com"}"#) )] #[case::structured_input( None, - Some(json!([{"role": "user", "content": [ - {"type": "input_text", "text": "What is the capital of France?"}, - {"type": "input_text", "text": "Answer briefly."}, - {"type": "input_image", "image_url": "https://example.com/paris.png"}, - ]}])), - Some("What is the capital of France?\nAnswer briefly."), -)] -#[case::model_objects_after_dump( + Some(r#"[{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}, {"type": "input_text", "text": "Answer briefly."}, {"type": "input_image", "image_url": "https://example.com/paris.png"}]}]"#), + Some(r#"What is the capital of France? +Answer briefly. +{"type":"input_image","image_url":"https://example.com/paris.png"}"#), +)] +#[case::message_items_with_ids( None, - Some(json!([ - {"content": [{"text": "model dump prompt"}]}, - {"content": [{"output_text": "dict prompt"}]}, - {"content": [{"input_text": "inline prompt"}]}, - {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, - ])), - Some("model dump prompt\ndict prompt\ninline prompt"), + Some(r#"[{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "hi", "annotations": []}]}]"#), + Some(r#"hi"#), )] #[case::object_content( None, - Some(json!({"content": [{"text": "object content prompt"}]})), - Some("object content prompt"), + Some(r#"{"content": [{"text": "object content prompt"}]}"#), + Some(r#"object content prompt"#) )] -#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))] -#[case::null_content_uses_text_keys( +#[case::string_content(None, Some(r#"{"content": " inline "}"#), Some(r#"inline"#))] +#[case::unknown_keys_kept( None, - Some(json!({"content": null, "output": "tool output"})), - Some("tool output"), + Some(r#"{"text": " ", "input_text": "fallback prompt"}"#), + Some(r#"{"text":" ","input_text":"fallback prompt"}"#) )] -#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)] -#[case::text_key_precedence( +#[case::nested_lists( None, - Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})), - Some("a"), -)] -#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))] -#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))] -#[case::non_string_text_keys_skipped( + Some(r#"[["a", [" b "]], "", "c"]"#), + Some( + r#"a +b +c"# + ) +)] +#[case::scalars_kept( None, - Some(json!({"text": 1, "output": "fallback"})), - Some("fallback"), -)] -#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] -#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] + Some(r#"[1, true, null, "kept"]"#), + Some( + r#"1 +true +kept"# + ) +)] +#[case::blank_text_parts(None, Some(r#"[{"type": "input_text", "text": " "}]"#), None)] #[case::responses_function_call( None, - Some(json!([ - {"role": "user", "content": "update the config"}, - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, - {"type": "function_call_output", "call_id": "c1", "output": "ok"}, - ])), - Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), + Some(r#"[{"role": "user", "content": "update the config"}, {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}]"#), + Some(r#"update the config +{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\"path\":\"a.yaml\"}"} +{"type":"function_call_output","call_id":1,"output":"ok"}"#), )] #[case::responses_structured_function_call_output( None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, - ])), - Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), -)] -#[case::responses_multi_part_output_joined_by_lines( - None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [ - {"type": "input_text", "text": " line one "}, - {"type": "input_text", "text": "line two"}, - ]}, - ])), - Some(r#"{"name":"run","arguments":"{}"} -{"result_of_call":1,"output":"line one\nline two"}"#), + Some(r#"[{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": " denied "}]}]"#), + Some(r#"{"type":"function_call","call_id":1,"name":"write_file","arguments":"{}"} +{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":" denied "}]}"#), )] #[case::responses_parallel_outputs_tagged_with_their_call( None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, - {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, - {"type": "function_call_output", "call_id": "c2", "output": "B"}, - {"type": "function_call_output", "call_id": "c1", "output": "A"}, - ])), - Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), -)] -#[case::responses_unknown_call_id_encodes_a_null_position( + Some(r#"[{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}]"#), + Some(r#"{"type":"function_call","call_id":1,"name":"read","arguments":"a"} +{"type":"function_call","call_id":2,"name":"read","arguments":"b"} +{"type":"function_call_output","call_id":2,"output":"B"} +{"type":"function_call_output","call_id":1,"output":"A"}"#), +)] +#[case::responses_reasoning_digested( None, - Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), - Some(r#"{"result_of_call":null,"output":"orphan"}"#), + Some(r#"[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAAB"}]"#), + Some(r#"{"type":"reasoning","id":1,"encrypted_content":"sha256:010b9eb723d8d1e2"}"#), )] fn prompt_from_context_matches_python( - #[case] messages: Option, - #[case] input: Option, + #[case] messages: Option<&str>, + #[case] input: Option<&str>, #[case] expected: Option<&str>, ) { - assert_eq!( - prompt_from_context(&context(messages, input)).as_deref(), - expected - ); -} - -/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank -/// text key falls through to the next one. -#[rstest] -#[case::blank_text_falls_through( - json!({"text": " ", "input_text": "fallback prompt"}), - "fallback prompt", -)] -fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) { - assert_eq!( - prompt_from_context(&context(None, Some(input))).as_deref(), - Some(expected) - ); -} - -/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from -/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and -/// strip also removes the ASCII information separators. -#[rstest] -#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)] -#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")] -#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")] -#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")] -#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")] -#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")] -fn python_serialization_edge_cases( - #[case] messages: Value, - #[case] input: Option, - #[case] expected: &str, -) { - let actual = match input { - Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(), - None => str_from_messages(messages.as_array().unwrap()), - }; - assert_eq!(actual, expected); + let parse = |value: &str| serde_json::from_str::(value).unwrap(); + let context = context(messages.map(parse), input.map(parse)); + assert_eq!(prompt_from_context(&context).as_deref(), expected); } #[rstest] diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 827fa91398fe..9bf889899588 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,9 +22,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages_with_tools, - tool_call_ordinals, - tool_call_str, - tool_result_str, + get_str_from_responses_input, ) from litellm.types.utils import EmbeddingResponse @@ -271,101 +269,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: if "input" not in kwargs: return None - responses_input: Final = kwargs.get("input") - prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) - return prompt or None - - @classmethod - def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: - prompt_parts: Final[list[str]] = [] - cls._collect_responses_input_text(value, prompt_parts, call_ordinals) - return "\n".join(prompt_parts).strip() - - @classmethod - def _collect_responses_input_text( - cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) - if value is None: - return - - if isinstance(value, str): - stripped_value: Final = value.strip() - if stripped_value: - prompt_parts.append(stripped_value) - return - - if isinstance(value, (list, tuple)): - for item in value: - cls._collect_responses_input_text(item, prompt_parts, call_ordinals) - return - - if isinstance(value, dict): - content = value.get("content") - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) - return - - content = getattr(value, "content", None) - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - for text_key in ("text", "output", "input_text", "output_text"): - text_value = getattr(value, text_key, None) - if isinstance(text_value, str): - stripped_text = text_value.strip() - if stripped_text: - prompt_parts.append(stripped_text) - return - - @classmethod - def _collect_responses_text_fields( - cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - for text_key in ("text", "output", "input_text", "output_text"): - text_value = value.get(text_key) - if isinstance(text_value, (list, tuple)): - cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) - return - if isinstance(text_value, str) and (stripped_text := text_value.strip()): - prompt_parts.append(stripped_text) - return - - @classmethod - def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: - items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () - dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) - return tool_call_ordinals( - item.get("call_id") - for item in dumped_items - if isinstance(item, dict) and item.get("type") == "function_call" - ) - - @classmethod - def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: - if not isinstance(value, dict): - return value - if value.get("type") == "function_call": - return tool_call_str(value.get("name"), value.get("arguments")) - if value.get("type") != "function_call_output": - return value - return tool_result_str( - value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) - ) - - @staticmethod - def _coerce_response_input_value(value: object) -> object: - model_dump: Final = getattr(value, "model_dump", None) - if callable(model_dump): - return model_dump() - dict_method: Final = getattr(value, "dict", None) - if callable(dict_method): - return dict_method() - return value + return get_str_from_responses_input(kwargs.get("input")) def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 0c52fae86d79..8e75ac4034eb 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,6 +2,7 @@ Common utility functions used for translating messages across providers """ +import hashlib import io import json import mimetypes @@ -195,93 +196,148 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text +_PLAIN_TYPES: Final = frozenset({"text", "input_text", "output_text", "message"}) +_PLAIN_KEYS: Final = frozenset({"role", "type", "text", "content", "status"}) +_TOOL_ROLES: Final = frozenset({"tool", "function"}) +_IGNORED_KEYS: Final = frozenset({"cache_control"}) +_CALL_ID_KEYS: Final = frozenset({"id", "call_id", "tool_use_id", "tool_call_id"}) +_OPAQUE_KEYS: Final = frozenset({"data", "file_data", "signature", "encrypted_content"}) + + def get_str_from_messages_with_tools(messages: object) -> str: """ - ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns - that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) - produce different text. Each result is encoded with the position of the call it answers, since call ids are - random per session + Semantic-cache prompt for chat-style ``messages``. Plain text parts stay plain text so text-only prompts embed + as they always have. Every other part (tool calls and results, images, unknown block types, extra fields) is kept + as compact JSON, so a new request shape changes the prompt instead of silently vanishing from it """ message_mappings: Final = tuple(_str_mappings(messages)) - call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) - return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) + call_ordinals: Final = _call_id_ordinals(message_mappings) + return "".join(_message_prompt(message, call_ordinals) for message in message_mappings) -def tool_call_str(name: object, arguments: object) -> str: - return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' +def get_str_from_responses_input(responses_input: object) -> str | None: + """``get_str_from_messages_with_tools`` for Responses API ``input``, one stripped line per part""" + call_ordinals: Final = _call_id_ordinals(responses_input) + prompt: Final = "\n".join(_responses_input_parts(responses_input, call_ordinals)).strip() + return prompt or None -def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: - ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None - return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' +def _message_prompt(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + role: Final = message.get("role") + if (isinstance(role, str) and role in _TOOL_ROLES) or not _is_plain(message): + return _compact_json(_normalized(message, call_ordinals)) + return _plain_prompt(message, call_ordinals) -def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: - string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) - return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) +def _plain_prompt(value: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + text: Final = value.get("text") + return (text if isinstance(text, str) else "") + _content_prompt(value.get("content"), call_ordinals) -def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=str) +def _content_prompt(content: object, call_ordinals: Mapping[str, int]) -> str: + value: Final = _plain_value(content) + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (list, tuple)): + return "".join(_content_prompt(item, call_ordinals) for item in value) + if _is_str_mapping(value) and _is_plain(value): + return _plain_prompt(value, call_ordinals) + return _compact_json(_normalized(value, call_ordinals)) -def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: - for message in messages: - yield from ( - block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" - ) - yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) +def _responses_input_parts(value: object, call_ordinals: Mapping[str, int]) -> Iterator[str]: + item: Final = _plain_value(value) + if item is None: + return + if isinstance(item, str): + yield from _stripped(item) + elif isinstance(item, (list, tuple)): + for child in item: + yield from _responses_input_parts(child, call_ordinals) + elif _is_str_mapping(item) and _is_plain(item): + text: Final = item.get("text") + yield from _stripped(text if isinstance(text, str) else "") + yield from _responses_input_parts(item.get("content"), call_ordinals) + else: + yield _compact_json(_normalized(item, call_ordinals)) + + +def _stripped(text: str) -> Iterator[str]: + stripped: Final = text.strip() + if stripped: + yield stripped -def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - content: Final = _content_str_with_tools(message.get("content"), call_ordinals) +def _is_plain(value: Mapping[str, object]) -> bool: + value_type: Final = value.get("type") + text: Final = value.get("text") return ( - ( - tool_result_str(message.get("tool_call_id"), call_ordinals, content) - if message.get("role") == "tool" - else content - ) - + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) - + extract_search_results_text(message.get("search_results")) + (value_type is None or (isinstance(value_type, str) and value_type in _PLAIN_TYPES)) + and (text is None or isinstance(text, str)) + and all(key in _PLAIN_KEYS or key in _CALL_ID_KEYS for key in _kept_keys(value)) ) -def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: - if isinstance(content, str): - return content - return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) +def _kept_keys(value: Mapping[str, object]) -> Iterator[str]: + return (key for key, item in value.items() if key not in _IGNORED_KEYS and not _is_empty(item)) -def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - block_type: Final = block.get("type") - if block_type == "tool_use": - return tool_call_str(block.get("name"), block.get("input")) - if block_type == "tool_result": - return tool_result_str( - block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) - ) - text: Final = block.get("text") - return text if isinstance(text, str) else "" +def _normalized(value: object, call_ordinals: Mapping[str, int], key: str | None = None) -> object: + item: Final = _plain_value(value) + if _is_str_mapping(item): + return MappingProxyType({kept: _normalized(item[kept], call_ordinals, kept) for kept in _kept_keys(item)}) + if isinstance(item, (list, tuple)): + return tuple(_normalized(child, call_ordinals) for child in item) + if not isinstance(item, str): + return item + if key in _CALL_ID_KEYS: + return call_ordinals.get(item, item) + if key in _OPAQUE_KEYS or item.startswith("data:"): + return f"sha256:{hashlib.sha256(item.encode('utf-8', 'surrogatepass')).hexdigest()[:16]}" + return item -def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: - function: Final = _as_str_mapping(tool_call.get("function")) - if function is None: - return tool_call_str(None, None) - return tool_call_str(function.get("name"), function.get("arguments")) +def _call_id_ordinals(value: object) -> Mapping[str, int]: + return MappingProxyType( + {call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(_call_ids(value)), start=1)} + ) -def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: - items: Final = values if isinstance(values, (list, tuple)) else () - return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) +def _call_ids(value: object, key: str | None = None) -> Iterator[str]: + item: Final = _plain_value(value) + if isinstance(item, str): + if key in _CALL_ID_KEYS: + yield item + elif _is_str_mapping(item): + for child_key, child in item.items(): + if child_key not in _IGNORED_KEYS: + yield from _call_ids(child, child_key) + elif isinstance(item, (list, tuple)): + for child in item: + yield from _call_ids(child) -def _as_str_mapping(value: object) -> Mapping[str, object] | None: - if isinstance(value, BaseModel): - return value.model_dump() - if _is_str_mapping(value): - return value - return None +def _is_empty(value: object) -> bool: + return value is None or (isinstance(value, (str, list, tuple, Mapping)) and not value) + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), default=_json_default) + + +def _json_default(value: object) -> object: + return dict(value) if isinstance(value, MappingProxyType) else str(value) + + +def _plain_value(value: object) -> object: + return value.model_dump() if isinstance(value, BaseModel) else value + + +def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: + items: Final = values if isinstance(values, (list, tuple)) else () + return (mapping for item in items if _is_str_mapping(mapping := _plain_value(item))) def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 3359796ec84d..6040ae2b214c 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -531,7 +531,7 @@ def test_redis_semantic_cache_get_cache_uses_responses_string_input(): ) -def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): +def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_responses_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) @@ -561,7 +561,7 @@ def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): ) redis_semantic_cache.llmcache.store.assert_called_once_with( - "What is the capital of France?\nAnswer briefly.", + 'What is the capital of France?\nAnswer briefly.\n{"type":"input_image","image_url":"https://example.com/paris.png"}', "{'content': 'Paris'}", vector=[0.1, 0.2, 0.3], filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, @@ -593,10 +593,11 @@ def turn(command: str) -> list[dict[str, object]]: ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' + 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}' + '{"type":"tool_result","tool_use_id":1,"content":"ok"}' ) - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) != ( + RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) ) @@ -612,7 +613,8 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ) assert prompt == ( - 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' + 'update the config\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}' + '\n{"type":"function_call_output","call_id":1,"output":"ok"}' ) @@ -632,30 +634,13 @@ def prompt_for(output_text: str) -> str | None: ] ) - expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' - assert ( - prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' + assert prompt_for("wrote 5 bytes") == ( + 'write hello\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n' + '{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":"wrote 5 bytes"}]}' ) assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") -def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - { - "type": "function_call_output", - "call_id": "c1", - "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], - }, - ] - ) - - assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' - - def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -670,80 +655,63 @@ def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | N ) assert prompt_for("c2", "c1") == ( - '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' - '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' + '{"type":"function_call","call_id":1,"name":"read","arguments":"a"}\n' + '{"type":"function_call","call_id":2,"name":"read","arguments":"b"}\n' + '{"type":"function_call_output","call_id":2,"output":"empty"}\n' + '{"type":"function_call_output","call_id":1,"output":"secret"}' ) assert prompt_for("c1", "c2") != prompt_for("c2", "c1") -def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): +def test_redis_semantic_cache_prompt_extraction_replaces_encrypted_reasoning_with_a_digest(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ModelDumpInput: - def model_dump(self): - return {"content": [{"text": "model dump prompt"}]} - - class DictInput: - def dict(self): - return {"content": [{"output_text": "dict prompt"}]} + def prompt_for(encrypted_content: str) -> str | None: + return RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content}] + ) - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - ModelDumpInput(), - DictInput(), - {"content": [{"input_text": "inline prompt"}]}, - {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, - ] - ) + blob = "gAAAAB" * 100 + prompt = prompt_for(blob) + assert prompt is not None + assert blob not in prompt + assert prompt.startswith('{"type":"reasoning","id":1,"encrypted_content":"sha256:') + assert prompt != prompt_for(blob + "x") - assert prompt == "model dump prompt\ndict prompt\ninline prompt" +def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): + from pydantic import BaseModel -def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - assert RedisSemanticCache._get_prompt_from_kwargs() is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None - assert ( - RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "input_image", "image_url": "https://example.com"}] - ) - is None - ) - - -def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache + class ResponseInputMessage(BaseModel): + content: list[dict[str, str]] prompt = RedisSemanticCache._get_prompt_from_kwargs( - input={"text": " ", "input_text": "fallback prompt"} + input=[ + ResponseInputMessage(content=[{"type": "input_text", "text": "model dump prompt"}]), + {"content": [{"type": "output_text", "text": "inline prompt", "annotations": []}]}, + ] ) - assert prompt == "fallback prompt" + assert prompt == "model dump prompt\ninline prompt" -def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): +def test_redis_semantic_cache_prompt_extraction_keeps_image_only_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ResponseInput: - text = " " - input_text = "fallback prompt" - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + assert RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "input_image", "image_url": "https://example.com/a.png"}] + ) == '{"type":"input_image","image_url":"https://example.com/a.png"}' - assert prompt == "fallback prompt" - -def test_redis_semantic_cache_prompt_extraction_handles_object_content(): +def test_redis_semantic_cache_prompt_extraction_returns_none_without_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ResponseInput: - content = [{"text": "object content prompt"}] - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) - - assert prompt == "object content prompt" + assert RedisSemanticCache._get_prompt_from_kwargs() is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_text", "text": " "}]) is None def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 59663de68d05..468bc71f95dd 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -3,6 +3,7 @@ import json import os import sys +from collections.abc import Callable from typing import Final import pytest @@ -2023,12 +2024,12 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], }, ], - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', - id="anthropic-tool-use-name-and-input-without-id", + 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}', + id="anthropic-tool-use-with-its-call-position", ), pytest.param( [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - 'fix the failing test{"result_of_call":null,"output":"calc.py"}', + 'fix the failing test{"type":"tool_result","tool_use_id":1,"content":"calc.py"}', id="anthropic-string-tool-result", ), pytest.param( @@ -2045,7 +2046,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ], } ], - '{"result_of_call":null,"output":"ab"}next', + '{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next', id="anthropic-nested-text-tool-result-then-text", ), pytest.param( @@ -2061,9 +2062,11 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], - 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' - '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', - id="openai-tool-calls-in-order-before-tool-result", + 'fix the failing test{"role":"assistant","content":"writing","tool_calls":[' + '{"id":1,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}},' + '{"id":2,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}}]}' + '{"role":"tool","tool_call_id":1,"content":"ok"}', + id="openai-tool-calls-then-tool-result", ), pytest.param( [ @@ -2082,8 +2085,9 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ], }, ], - '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' - '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', + '{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}' + '{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}' + '{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}', id="anthropic-parallel-tool-results-tagged-with-their-call", ), pytest.param( @@ -2093,19 +2097,11 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "content": None, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + {"role": "tool", "tool_call_id": "call_0", "content": "C"}, ], - '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' - '{"result_of_call":2,"output":"C"}', - id="reused-call-ids-keep-first-position", + '{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}' + '{"role":"tool","tool_call_id":1,"content":"C"}', + id="null-content-dropped-from-encoded-message", ), pytest.param( [ @@ -2116,13 +2112,13 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "search_results": [{"source": "s", "title": "t"}], } ], - '{"result_of_call":null,"output":"small"}st', - id="tool-result-search-results-follow-the-encoded-result", + '{"role":"tool","tool_call_id":1,"content":"small","search_results":[{"source":"s","title":"t"}]}', + id="tool-result-keeps-search-results", ), pytest.param( - [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], - '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', - id="tool-output-cannot-forge-an-encoded-result", + [{"role": "tool", "content": 'x"}{"role":"tool","content":"y'}], + '{"role":"tool","content":"x\\"}{\\"role\\":\\"tool\\",\\"content\\":\\"y"}', + id="tool-output-is-escaped-even-without-a-call-id", ), pytest.param( [ @@ -2131,17 +2127,32 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], ) ], - '{"name":"read","arguments":"{}"}', + '{"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"read"},"id":1,"type":"function"}]}', id="openai-response-message-object", ), pytest.param( - [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], - '{"name":null,"arguments":null}', - id="malformed-tool-call-entries", + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + "hi", + id="cache-control-ignored", + ), + pytest.param( + [{"role": "user", "name": "alice", "content": "hi"}], + '{"role":"user","name":"alice","content":"hi"}', + id="extra-message-field-kept", + ), + pytest.param( + [{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}], + '{"type":"brand_new_block","payload":{"k":"v"}}!', + id="unknown-block-type-kept", + ), + pytest.param( + [{"role": "user", "content": [{"type": "text", "text": 3}]}], + '{"type":"text","text":3}', + id="non-string-text-kept", ), ], ) -def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: +def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object], expected: str) -> None: assert get_str_from_messages_with_tools(messages) == expected @@ -2150,25 +2161,54 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj [ pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), pytest.param( - [ - { - "role": "user", - "content": [ - {"type": "text", "text": "what is "}, - {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, - {"type": "text", "text": "this"}, - ], - } - ], - id="text-and-image-parts", + [{"role": "user", "content": [{"type": "text", "text": "what is "}, {"type": "text", "text": "this"}]}], + id="text-parts", ), pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: +def test_get_str_from_messages_with_tools_matches_get_str_from_messages_for_text(messages: list[object]) -> None: assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures +def _anthropic_image(data: str) -> list[object]: + return [{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "data": data}}]}] + + +def _openai_image(url: str) -> list[object]: + return [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}] + + +def _thinking(signature: str) -> list[object]: + return [{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": signature}]}] + + +@pytest.mark.parametrize( + ("build", "blob"), + [ + pytest.param(_anthropic_image, "iVBORw0KGgo" * 50, id="anthropic-base64-image"), + pytest.param(_openai_image, "data:image/png;base64," + "iVBORw0KGgo" * 50, id="openai-data-url-image"), + pytest.param(_thinking, "EqQBCkgIARABGAIiQL" * 20, id="thinking-signature"), + ], +) +def test_get_str_from_messages_with_tools_replaces_opaque_blobs_with_distinct_digests( + build: Callable[[str], list[object]], blob: str +) -> None: + prompt: Final = get_str_from_messages_with_tools(build(blob)) + + assert blob not in prompt + assert "sha256:" in prompt + assert len(prompt) < len(blob) + assert prompt == get_str_from_messages_with_tools(build(blob)) + assert prompt != get_str_from_messages_with_tools(build(blob + "A")) + + +def test_get_str_from_messages_with_tools_keeps_image_urls_distinct() -> None: + assert get_str_from_messages_with_tools(_openai_image("https://example.com/a.png")) != ( + get_str_from_messages_with_tools(_openai_image("https://example.com/b.png")) + ) + + def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: return [ { From 39efb5d9da8292844a9844e01d095702b11ecec4 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:15:34 +0000 Subject: [PATCH 10/29] test(code-quality): allow the bounded semantic cache prompt walkers in the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/code_coverage_tests/recursive_detector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 659dc438f2d2..e071e177f885 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -72,6 +72,10 @@ "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). + "_content_prompt", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). + "_responses_input_parts", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). + "_normalized", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). + "_call_ids", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. ] From 86c82b949b3da25461e72f317b0f54572ea5359f Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:33:41 +0000 Subject: [PATCH 11/29] test(rust): expect structured JSON for unknown fields in redis and valkey semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache-redis-semantic/tests/cache.rs | 4 ++-- litellm-rust/crates/cache-valkey-semantic/tests/cache.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs index fa4ac00e7674..dda6adc0998d 100644 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -509,7 +509,7 @@ fn tag_special_characters_are_escaped_in_search_filter( )] #[case::responses_output_text( SemanticCacheContext { input: Some(json!({"output_text": " result text "})), ..Default::default() }, - "result text" + "{\"output_text\":\" result text \"}" )] #[case::search_results( messages_context(vec![json!({ @@ -517,7 +517,7 @@ fn tag_special_characters_are_escaped_in_search_filter( "content": "question", "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], })]), - "questionsrctfound{\"a\":1}" + r#"{"role":"user","content":"question","search_results":[{"source":"src","title":"t","content":[{"text":"found"}],"citations":{"a":1}}]}"# )] #[case::empty_messages_fall_back_to_input( SemanticCacheContext { messages: Some(json!([])), input: Some(json!("fallback")), ..Default::default() }, diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs index 1c693d0875de..4ac45ed0b2aa 100644 --- a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs @@ -142,8 +142,8 @@ fn flat_dimension_info(dimension: &str) -> redis::Value { #[rstest] #[case::string_content(json!([{"content": "hello"}]), None, Some("hello"))] #[case::text_parts(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] -#[case::non_object_parts_are_skipped(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("hello"))] -#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] +#[case::string_parts_are_kept(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("rawhello"))] +#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"{"search_results":[{"source":"s","title":"t","content":[{"text":"c"}],"citations":["x"]}]}"#))] #[case::responses_string_input(json!([]), Some(json!(" hello ")), Some("hello"))] #[case::responses_item_input(json!([]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] #[case::blank_input(json!([]), Some(json!(" ")), None)] From 345bd97f708d2f1c234a51d72a38663fe56181e5 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:39:28 +0000 Subject: [PATCH 12/29] Revert "test(rust): expect structured JSON for unknown fields in redis and valkey semantic prompts" This reverts commit 86c82b949b3da25461e72f317b0f54572ea5359f. --- litellm-rust/crates/cache-redis-semantic/tests/cache.rs | 4 ++-- litellm-rust/crates/cache-valkey-semantic/tests/cache.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs index dda6adc0998d..fa4ac00e7674 100644 --- a/litellm-rust/crates/cache-redis-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-redis-semantic/tests/cache.rs @@ -509,7 +509,7 @@ fn tag_special_characters_are_escaped_in_search_filter( )] #[case::responses_output_text( SemanticCacheContext { input: Some(json!({"output_text": " result text "})), ..Default::default() }, - "{\"output_text\":\" result text \"}" + "result text" )] #[case::search_results( messages_context(vec![json!({ @@ -517,7 +517,7 @@ fn tag_special_characters_are_escaped_in_search_filter( "content": "question", "search_results": [{"source": "src", "title": "t", "content": [{"text": "found"}], "citations": {"a": 1}}], })]), - r#"{"role":"user","content":"question","search_results":[{"source":"src","title":"t","content":[{"text":"found"}],"citations":{"a":1}}]}"# + "questionsrctfound{\"a\":1}" )] #[case::empty_messages_fall_back_to_input( SemanticCacheContext { messages: Some(json!([])), input: Some(json!("fallback")), ..Default::default() }, diff --git a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs index 4ac45ed0b2aa..1c693d0875de 100644 --- a/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs +++ b/litellm-rust/crates/cache-valkey-semantic/tests/cache.rs @@ -142,8 +142,8 @@ fn flat_dimension_info(dimension: &str) -> redis::Value { #[rstest] #[case::string_content(json!([{"content": "hello"}]), None, Some("hello"))] #[case::text_parts(json!([{"content": [{"text": "hello"}, {"text": " world"}]}]), None, Some("hello world"))] -#[case::string_parts_are_kept(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("rawhello"))] -#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"{"search_results":[{"source":"s","title":"t","content":[{"text":"c"}],"citations":["x"]}]}"#))] +#[case::non_object_parts_are_skipped(json!([{"content": ["raw", {"text": "hello"}]}]), None, Some("hello"))] +#[case::search_results(json!([{"search_results": [{"source": "s", "title": "t", "content": [{"text": "c"}], "citations": ["x"]}]}]), None, Some(r#"stc["x"]"#))] #[case::responses_string_input(json!([]), Some(json!(" hello ")), Some("hello"))] #[case::responses_item_input(json!([]), Some(json!([{"content": "first"}, {"text": "second"}])), Some("first\nsecond"))] #[case::blank_input(json!([]), Some(json!(" ")), None)] From f8aad2fa29fae604b04eb9c9933101d5815a91f8 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:39:28 +0000 Subject: [PATCH 13/29] Revert "test(code-quality): allow the bounded semantic cache prompt walkers in the recursion check" This reverts commit 39efb5d9da8292844a9844e01d095702b11ecec4. --- tests/code_coverage_tests/recursive_detector.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index e071e177f885..659dc438f2d2 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -72,10 +72,6 @@ "_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible). "_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible). - "_content_prompt", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). - "_responses_input_parts", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). - "_normalized", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). - "_call_ids", # bounded by the nesting depth of the request messages or input it walks (a finite JSON tree, no cycles possible). "scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap. ] From ccca38f7aedbaf1a3e94a88f5d4f6f3b94f94855 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:39:28 +0000 Subject: [PATCH 14/29] Revert "feat(caching): embed every semantic cache prompt field except volatile ones" This reverts commit 5aed3ab3de3597dec22d3afcb33db7196f0a2a24. --- litellm-rust/Cargo.lock | 1 - .../cache-qdrant-semantic/tests/qdrant.rs | 8 +- litellm-rust/crates/cache/Cargo.toml | 1 - litellm-rust/crates/cache/src/semantic.rs | 386 ++++++++-------- litellm-rust/crates/cache/tests/semantic.rs | 435 +++++++++++------- litellm/caching/redis_semantic_cache.py | 100 +++- .../prompt_templates/common_utils.py | 174 +++---- .../unit/caching/test_redis_semantic_cache.py | 120 +++-- ...ore_utils_prompt_templates_common_utils.py | 126 ++--- 9 files changed, 742 insertions(+), 609 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 63b4c9352086..8d189c8c515a 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3426,7 +3426,6 @@ dependencies = [ "rstest", "serde", "serde_json", - "sha2 0.10.9", "thiserror 2.0.19", "tokio", ] diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index 2d94d7661463..7a27519c2691 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -225,7 +225,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { } #[rstest] -#[case::content_parts_keep_images_in_place( +#[case::content_parts_skip_images( json!([ {"role": "user", "content": "hello"}, { @@ -237,9 +237,9 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { ], }, ]), - r#"helloworld{"type":"image_url","image_url":{"url":"ignored"}}!"# + "helloworld!" )] -#[case::tool_message_with_search_results_kept_as_json( +#[case::search_results_and_compact_citations( json!([{ "role": "tool", "content": null, @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"{"role":"tool","search_results":[{"source":"source","title":"title","content":[{"text":"body"}],"citations":{"page":1,"section":"intro"}}]}"# + r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index 69b898adb11c..f18dbd9cb260 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [dependencies] serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } -sha2.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index db14abee664f..253efb477596 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -8,10 +8,9 @@ use std::{collections::HashMap, future::Future, io}; use serde::Serialize; use serde_json::{ - Map, Value, + Value, json, ser::{CharEscape, Formatter, Serializer}, }; -use sha2::{Digest, Sha256}; use crate::{BaseCache, Error, SemanticCacheContext}; @@ -84,242 +83,235 @@ impl Embedder for PreparedEmbedding { } } -const PLAIN_TYPES: [&str; 4] = ["text", "input_text", "output_text", "message"]; -const PLAIN_KEYS: [&str; 5] = ["role", "type", "text", "content", "status"]; -const TOOL_ROLES: [&str; 2] = ["tool", "function"]; -const IGNORED_KEYS: [&str; 1] = ["cache_control"]; -const CALL_ID_KEYS: [&str; 4] = ["id", "call_id", "tool_use_id", "tool_call_id"]; -const OPAQUE_KEYS: [&str; 4] = ["data", "file_data", "signature", "encrypted_content"]; - -/// `get_str_from_messages_with_tools`: plain text parts stay plain text, and every other part is -/// kept as compact JSON with call ids replaced by their position and opaque blobs by a digest. +/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, +/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the +/// position of the call it answers. pub fn str_from_messages(messages: &[Value]) -> String { - let messages: Vec<&Value> = messages - .iter() - .filter(|message| message.is_object()) - .collect(); - let call_ordinals = call_id_ordinals(messages.iter().copied()); - messages - .into_iter() - .filter_map(Value::as_object) - .map(|message| message_prompt(message, &call_ordinals)) - .collect() -} - -/// `get_str_from_responses_input`: `str_from_messages` for a Responses API `input`, one stripped -/// line per part. `None` when nothing is left. -fn str_from_responses_input(input: &Value) -> Option { - let call_ordinals = call_id_ordinals(std::iter::once(input)); - let mut parts = Vec::new(); - push_responses_input_parts(input, &call_ordinals, &mut parts); - let prompt = python_strip(&parts.join("\n")).to_owned(); - (!prompt.is_empty()).then_some(prompt) -} - -fn message_prompt(message: &Map, call_ordinals: &HashMap<&str, usize>) -> String { - let is_tool = message - .get("role") - .and_then(Value::as_str) - .is_some_and(|role| TOOL_ROLES.contains(&role)); - if is_tool || !is_plain(message) { - return compact_json(&normalized(message, call_ordinals)); + let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); + let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { + let block_ids = message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) + .filter_map(|block| block.get("id")); + let tool_call_ids = message + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tool_call| tool_call.get("id")); + block_ids.chain(tool_call_ids) + })); + let mut text = String::new(); + for message in messages { + if message.get("role").and_then(Value::as_str) == Some("tool") { + let mut output = String::new(); + push_content_text(&mut output, message.get("content"), &call_ordinals); + text.push_str(&tool_result_json( + message.get("tool_call_id"), + &call_ordinals, + &output, + )); + } else { + push_content_text(&mut text, message.get("content"), &call_ordinals); + } + if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { + for tool_call in tool_calls.iter().filter_map(Value::as_object) { + let function = tool_call.get("function"); + text.push_str(&tool_call_json( + function.and_then(|function| function.get("name")), + function.and_then(|function| function.get("arguments")), + )); + } + } + push_search_results_text(&mut text, message.get("search_results")); } - plain_prompt(message, call_ordinals) + text } -fn plain_prompt(value: &Map, call_ordinals: &HashMap<&str, usize>) -> String { - let mut prompt = value - .get("text") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned(); - push_content_prompt(&mut prompt, value.get("content"), call_ordinals); - prompt -} - -fn push_content_prompt( - prompt: &mut String, +/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. +fn push_content_text( + text: &mut String, content: Option<&Value>, call_ordinals: &HashMap<&str, usize>, ) { match content { - None | Some(Value::Null) => {} - Some(Value::String(text)) => prompt.push_str(text), - Some(Value::Array(items)) => { - for item in items { - push_content_prompt(prompt, Some(item), call_ordinals); + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(blocks)) => { + for block in blocks.iter().filter_map(Value::as_object) { + match block.get("type").and_then(Value::as_str) { + Some("tool_use") => { + text.push_str(&tool_call_json(block.get("name"), block.get("input"))); + } + Some("tool_result") => { + let mut output = String::new(); + push_content_text(&mut output, block.get("content"), call_ordinals); + text.push_str(&tool_result_json( + block.get("tool_use_id"), + call_ordinals, + &output, + )); + } + _ => { + if let Some(block_text) = block.get("text").and_then(Value::as_str) { + text.push_str(block_text); + } + } + } } } - Some(Value::Object(map)) if is_plain(map) => { - prompt.push_str(&plain_prompt(map, call_ordinals)); - } - Some(Value::Object(map)) => prompt.push_str(&compact_json(&normalized(map, call_ordinals))), - Some(other) => prompt.push_str(&compact_json(other)), + _ => {} } } -fn push_responses_input_parts( - value: &Value, - call_ordinals: &HashMap<&str, usize>, - parts: &mut Vec, -) { - match value { - Value::Null => {} - Value::String(text) => push_stripped(text, parts), - Value::Array(items) => { - for item in items { - push_responses_input_parts(item, call_ordinals, parts); - } - } - Value::Object(map) if is_plain(map) => { - push_stripped( - map.get("text").and_then(Value::as_str).unwrap_or_default(), - parts, - ); - if let Some(content) = map.get("content") { - push_responses_input_parts(content, call_ordinals, parts); - } - } - Value::Object(map) => parts.push(compact_json(&normalized(map, call_ordinals))), - other => parts.push(compact_json(other)), +/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. +fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { + let mut ordinals = HashMap::new(); + for call_id in call_ids.filter_map(Value::as_str) { + let next = ordinals.len() + 1; + ordinals.entry(call_id).or_insert(next); } + ordinals } -fn push_stripped(text: &str, parts: &mut Vec) { - let stripped = python_strip(text); - if !stripped.is_empty() { - parts.push(stripped.to_owned()); - } +/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result +/// answers no known call. +fn tool_result_json( + call_id: Option<&Value>, + call_ordinals: &HashMap<&str, usize>, + output: &str, +) -> String { + let ordinal = call_id + .and_then(Value::as_str) + .and_then(|call_id| call_ordinals.get(call_id)); + format!( + "{{\"result_of_call\":{},\"output\":{}}}", + compact_json(&json!(ordinal)), + compact_json(&Value::String(output.to_owned())), + ) } -/// `_is_plain`: a text-only part whose every kept key is a plain or call id key. -fn is_plain(value: &Map) -> bool { - let plain_type = match value.get("type") { - None | Some(Value::Null) => true, - Some(Value::String(value_type)) => PLAIN_TYPES.contains(&value_type.as_str()), - Some(_) => false, - }; - let plain_text = matches!( - value.get("text"), - None | Some(Value::Null | Value::String(_)) - ); - plain_type - && plain_text - && kept_entries(value).all(|(key, _)| { - PLAIN_KEYS.contains(&key.as_str()) || CALL_ID_KEYS.contains(&key.as_str()) - }) +/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. +fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { + compact_json(&json!({ + "name": name.unwrap_or(&Value::Null), + "arguments": arguments.unwrap_or(&Value::Null), + })) } -fn kept_entries(value: &Map) -> impl Iterator { - value - .iter() - .filter(|(key, item)| !IGNORED_KEYS.contains(&key.as_str()) && !is_empty(item)) +/// The messages prompt Qdrant embeds: `None` when the request carries no messages. +pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { + let messages = context.messages.as_ref()?.as_array()?; + (!messages.is_empty()).then(|| str_from_messages(messages)) } -fn is_empty(value: &Value) -> bool { - match value { - Value::Null => true, - Value::String(text) => text.is_empty(), - Value::Array(items) => items.is_empty(), - Value::Object(map) => map.is_empty(), - Value::Bool(_) | Value::Number(_) => false, +/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a +/// Responses API `input`. `None` when neither yields a prompt. +pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { + if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) + && !messages.is_empty() + { + return Some(str_from_messages(messages)); } + let input = context.input.as_ref()?; + let call_ordinals = tool_call_ordinals( + input + .as_array() + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) + .filter_map(|item| item.get("call_id")), + ); + let mut parts = Vec::new(); + collect_input_text(input, &mut parts, &call_ordinals); + let prompt = python_strip(&parts.join("\n")).to_owned(); + (!prompt.is_empty()).then_some(prompt) } -/// `_normalized`: the part minus ignored and empty entries, with call ids replaced by their -/// position and opaque blobs by `sha256:` plus the first 16 hex digits of their digest. -fn normalized(value: &Map, call_ordinals: &HashMap<&str, usize>) -> Value { - Value::Object( - kept_entries(value) - .map(|(key, item)| { - ( - key.clone(), - normalized_value(item, call_ordinals, Some(key)), - ) - }) - .collect(), - ) +/// `extract_search_results_text`. +fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { + let Some(Value::Array(results)) = search_results else { + return; + }; + for result in results.iter().filter_map(Value::as_object) { + for key in ["source", "title"] { + if let Some(value) = result.get(key).and_then(Value::as_str) { + text.push_str(value); + } + } + if let Some(Value::Array(content)) = result.get("content") { + for block in content.iter().filter_map(Value::as_object) { + if let Some(value) = block.get("text").and_then(Value::as_str) { + text.push_str(value); + } + } + } + if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) { + text.push_str(&compact_json(citations)); + } + } } -fn normalized_value( +fn collect_input_text( value: &Value, + parts: &mut Vec, call_ordinals: &HashMap<&str, usize>, - key: Option<&str>, -) -> Value { +) { match value { - Value::Object(map) => normalized(map, call_ordinals), - Value::Array(items) => Value::Array( - items - .iter() - .map(|item| normalized_value(item, call_ordinals, None)) - .collect(), - ), - Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => call_ordinals - .get(text.as_str()) - .map_or_else(|| value.clone(), |ordinal| Value::from(*ordinal)), - Value::String(text) - if key.is_some_and(|key| OPAQUE_KEYS.contains(&key)) || text.starts_with("data:") => - { - Value::String(digest(text)) + Value::String(text) => { + push_trimmed(text, parts); + } + Value::Array(items) => { + for item in items { + collect_input_text(item, parts, call_ordinals); + } } - other => other.clone(), - } -} - -fn digest(text: &str) -> String { - let hash = Sha256::digest(text.as_bytes()); - let hex: String = hash[..8].iter().map(|byte| format!("{byte:02x}")).collect(); - format!("sha256:{hex}") -} - -/// `_call_id_ordinals`: the 1-based position of each distinct call id string, first seen first. -fn call_id_ordinals<'a>(values: impl Iterator) -> HashMap<&'a str, usize> { - let mut ids = Vec::new(); - for value in values { - collect_call_ids(value, None, &mut ids); - } - let mut ordinals = HashMap::new(); - for id in ids { - let next = ordinals.len() + 1; - ordinals.entry(id).or_insert(next); - } - ordinals -} - -fn collect_call_ids<'a>(value: &'a Value, key: Option<&str>, ids: &mut Vec<&'a str>) { - match value { - Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => ids.push(text), Value::Object(map) => { - for (child_key, child) in map { - if !IGNORED_KEYS.contains(&child_key.as_str()) { - collect_call_ids(child, Some(child_key), ids); + if map.get("type").and_then(Value::as_str) == Some("function_call") { + parts.push(tool_call_json(map.get("name"), map.get("arguments"))); + return; + } + if map.get("type").and_then(Value::as_str) == Some("function_call_output") { + let mut output_parts = Vec::new(); + if let Some(output) = map.get("output") { + collect_input_text(output, &mut output_parts, call_ordinals); } + parts.push(tool_result_json( + map.get("call_id"), + call_ordinals, + python_strip(&output_parts.join("\n")), + )); + return; } - } - Value::Array(items) => { - for item in items { - collect_call_ids(item, None, ids); + if let Some(content) = map.get("content").filter(|content| !content.is_null()) { + collect_input_text(content, parts, call_ordinals); + return; + } + for key in ["text", "output", "input_text", "output_text"] { + match map.get(key) { + Some(nested @ Value::Array(_)) => { + collect_input_text(nested, parts, call_ordinals); + return; + } + Some(Value::String(text)) if push_trimmed(text, parts) => return, + _ => {} + } } } _ => {} } } -/// The messages prompt Qdrant embeds: `None` when the request carries no messages. -pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { - let messages = context.messages.as_ref()?.as_array()?; - (!messages.is_empty()).then(|| str_from_messages(messages)) -} - -/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a -/// Responses API `input`. `None` when neither yields a prompt. -pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { - if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) - && !messages.is_empty() - { - return Some(str_from_messages(messages)); +/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left. +fn push_trimmed(text: &str, parts: &mut Vec) -> bool { + let trimmed = python_strip(text); + if trimmed.is_empty() { + return false; } - str_from_responses_input(context.input.as_ref()?) + parts.push(trimmed.to_owned()); + true } /// `str.strip()`: Python's whitespace also covers the ASCII information separators. diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index c4a1bfb2d567..af87e6e0164a 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -16,115 +16,168 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex } #[rstest] -#[case::empty(r#"[]"#, r#""#)] -#[case::string_content(r#"[{"role": "user", "content": "hello"}]"#, r#"hello"#)] +#[case::empty(json!([]), "")] +#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")] #[case::concatenates_messages( - r#"[{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]"#, - r#"be brief. hello"# + json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]), + "be brief. hello", )] #[case::text_parts( - r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "text", "text": "this?"}]}]"#, - r#"What is this?"#, -)] -#[case::image_part_kept_in_place( - r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "this?"}]}]"#, - r#"What is {"type":"image_url","image_url":{"url":"https://example.com/a.png"}}this?"#, + json!([{"role": "user", "content": [ + {"type": "text", "text": "What is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this?"}, + ]}]), + "What is this?", )] #[case::missing_null_and_empty_content( - r#"[{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]"#, - r#""#, -)] -#[case::non_object_messages_skipped( - r#"["junk", 3, {"role": "user", "content": "kept"}]"#, - r#"kept"# -)] -#[case::cache_control_ignored( - r#"[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}]"#, - r#"hi"#, -)] -#[case::extra_message_field_kept( - r#"[{"role": "user", "name": "alice", "content": "hi"}]"#, - r#"{"role":"user","name":"alice","content":"hi"}"# -)] -#[case::unknown_block_type_kept( - r#"[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}]"#, - r#"{"type":"brand_new_block","payload":{"k":"v"}}!"#, -)] -#[case::non_string_text_kept( - r#"[{"role": "user", "content": [{"type": "text", "text": 3}]}]"#, - r#"{"type":"text","text":3}"# -)] -#[case::scalar_content_kept( - r#"[{"role": "user", "content": 7}, {"role": "user", "content": [1.5, true, "x"]}]"#, - r#"71.5truex"# -)] -#[case::search_results_kept( - r#"[{"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": [{"type": "text", "text": "hidden payload"}], "citations": {"enabled": true}}]}]"#, - r#"{"role":"tool","content":"small","search_results":[{"source":"s","title":"t","content":[{"type":"text","text":"hidden payload"}],"citations":{"enabled":true}}]}"#, -)] -#[case::anthropic_tool_use( - r#"[{"role": "user", "content": "fix the failing test"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}]}]"#, - r#"fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}"#, + json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), + "", +)] +#[case::search_results_hidden_behind_small_content( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, + ]}]), + r#"{"result_of_call":null,"output":"small"}sthidden payload"#, +)] +#[case::title_only_search_result( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "long title", "content": []}, + ]}]), + r#"{"result_of_call":null,"output":"small"}slong title"#, +)] +#[case::search_results_without_content( + json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), + r#"{"result_of_call":null,"output":""}st"#, +)] +#[case::search_result_fields_in_python_order( + json!([{"role": "tool", "content": "c", "search_results": [ + {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, + {"source": "s2"}, + ]}]), + r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, +)] +#[case::null_citations_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + {"source": "s", "citations": null}, + ]}]), + r#"{"result_of_call":null,"output":"c"}s"#, +)] +#[case::non_string_and_non_object_entries_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + "junk", + {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, + ]}]), + r#"{"result_of_call":null,"output":"c"}kept"#, +)] +#[case::non_list_search_results_skipped( + json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), + r#"{"result_of_call":null,"output":"c"}"#, +)] +#[case::citations_compact_in_insertion_order( + json!([{"role": "tool", "search_results": [ + {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, + ]}]), + r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, +)] +#[case::citations_ensure_ascii( + json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), + r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, +)] +#[case::citations_astral_chars_as_surrogate_pairs( + json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), + r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, +)] +#[case::citations_escapes( + json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), + r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, +)] +#[case::citations_large_float_exponent( + json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), + r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, +)] +#[case::citations_scalars( + json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), + r#"{"result_of_call":null,"output":""}false3"#, +)] +#[case::anthropic_tool_use_name_and_input_without_id( + json!([ + {"role": "user", "content": "fix the failing test"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, + ]}, + ]), + r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, )] #[case::anthropic_string_tool_result( - r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}]"#, - r#"{"type":"tool_result","tool_use_id":1,"content":"calc.py"}"#, + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, + ]}]), + r#"{"result_of_call":null,"output":"calc.py"}"#, )] #[case::anthropic_nested_text_tool_result_then_text( - r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "next"}]}]"#, - r#"{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next"#, -)] -#[case::openai_tool_calls_then_tool_result( - r#"[{"role": "assistant", "content": "writing", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}]"#, - r#"{"role":"assistant","content":"writing","tool_calls":[{"id":1,"type":"function","function":{"name":"write","arguments":"{\"path\": \"a\"}"}},{"id":2,"type":"function","function":{"name":"write","arguments":"{\"path\": \"b\"}"}}]}{"role":"tool","tool_call_id":1,"content":"ok"}"#, + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b"}, + ]}, + {"type": "text", "text": "next"}, + ]}]), + r#"{"result_of_call":null,"output":"ab"}next"#, +)] +#[case::openai_tool_calls_in_order_before_tool_result( + json!([ + {"role": "assistant", "content": "writing", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, + {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, + ]}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + ]), + r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, )] #[case::anthropic_parallel_tool_results_tagged_with_their_call( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "B"}, {"type": "tool_result", "tool_use_id": "t1", "content": "A"}]}]"#, - r#"{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}"#, + json!([ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, + {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, + {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, + ]}, + ]), + r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, )] #[case::reused_call_ids_keep_first_position( - r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}]"#, - r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"b","arguments":"{}"}},{"id":2,"type":"function","function":{"name":"c","arguments":"{}"}}]}{"role":"tool","tool_call_id":2,"content":"C"}"#, -)] -#[case::tool_output_cannot_forge_a_record( - r#"[{"role": "tool", "content": "x\"}{\"role\":\"tool\",\"content\":\"y"}]"#, - r#"{"role":"tool","content":"x\"}{\"role\":\"tool\",\"content\":\"y"}"# + json!([ + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, + ]}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + ]), + r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, +)] +#[case::unknown_call_ids_encode_a_null_position( + json!([ + {"role": "tool", "tool_call_id": "c9", "content": "ok"}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, + ]), + r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, +)] +#[case::tool_output_cannot_forge_an_encoded_result( + json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), + r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, )] #[case::malformed_tool_call_entries( - r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]"#, - r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function"},"junk"]}"#, -)] -#[case::anthropic_base64_image_digested( - r#"[{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, - r#"{"type":"image","source":{"type":"base64","media_type":"image/png","data":"sha256:b1a66f8de3276946"}}"#, -)] -#[case::openai_data_url_digested( - r#"[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, - r#"{"type":"image_url","image_url":{"url":"sha256:19c7f028a60a33b8"}}"#, -)] -#[case::thinking_signature_digested( - r#"[{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": "EqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARAB"}, {"type": "text", "text": "done"}]}]"#, - r#"{"type":"thinking","thinking":"hmm","signature":"sha256:b20cbbd82e4b3791"}done"#, -)] -#[case::non_ascii_escaped( - r#"[{"role": "tool", "content": "caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}]"#, - r#"{"role":"tool","content":"caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}"# + json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), + r#"{"name":null,"arguments":null}"#, )] -#[case::non_ascii_plain_text_kept_raw( - r#"[{"role": "user", "content": "caf\u00e9 \ud83d\ude00"}]"#, - r#"café 😀"# -)] -#[case::floats_in_python_repr( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {"a": [1e+20, 1.0, 1.5e-07, 0.0001, 1e+16, -2.5, 3]}}]}]"#, - r#"{"type":"tool_use","id":1,"name":"n","input":{"a":[1e+20,1.0,1.5e-07,0.0001,1e+16,-2.5,3]}}"#, -)] -#[case::empty_fields_dropped( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {}, "extra": [], "note": ""}]}]"#, - r#"{"type":"tool_use","id":1,"name":"n"}"#, -)] -fn str_from_messages_matches_python(#[case] messages: &str, #[case] expected: &str) { - let messages: Vec = serde_json::from_str(messages).unwrap(); - assert_eq!(str_from_messages(&messages), expected); +fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { + assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); } #[rstest] @@ -143,109 +196,167 @@ fn prompt_from_messages_reads_messages_only( #[rstest] #[case::prefers_messages( - Some(r#"[{"content": "message prompt"}]"#), - Some(r#""responses prompt""#), - Some(r#"message prompt"#) + Some(json!([{"content": "message prompt"}])), + Some(json!("responses prompt")), + Some("message prompt"), )] #[case::empty_messages_fall_back_to_input( - Some(r#"[]"#), - Some(r#""responses prompt""#), - Some(r#"responses prompt"#) + Some(json!([])), + Some(json!("responses prompt")), + Some("responses prompt"), )] #[case::messages_without_text_keep_an_empty_prompt( - Some(r#"[{"content": null}]"#), - Some(r#""x""#), - Some(r#""#) + Some(json!([{"content": null}])), + Some(json!("x")), + Some(""), )] -#[case::null_input(None, None, None)] -#[case::blank_string(None, Some(r#"" ""#), None)] +#[case::nothing(None, None, None)] +#[case::null_input(None, Some(Value::Null), None)] +#[case::blank_string(None, Some(json!(" ")), None)] #[case::trimmed_string( None, - Some(r#"" What is the capital of France?\n""#), - Some(r#"What is the capital of France?"#) + Some(json!(" What is the capital of France?\n")), + Some("What is the capital of France?"), )] -#[case::strip_information_separators(None, Some(r#""\u001ca\u001f""#), Some(r#"a"#))] -#[case::image_only_kept( +#[case::image_only( + None, + Some(json!([{"type": "input_image", "image_url": "https://example.com"}])), None, - Some(r#"[{"type": "input_image", "image_url": "https://example.com"}]"#), - Some(r#"{"type":"input_image","image_url":"https://example.com"}"#) )] #[case::structured_input( None, - Some(r#"[{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}, {"type": "input_text", "text": "Answer briefly."}, {"type": "input_image", "image_url": "https://example.com/paris.png"}]}]"#), - Some(r#"What is the capital of France? -Answer briefly. -{"type":"input_image","image_url":"https://example.com/paris.png"}"#), -)] -#[case::message_items_with_ids( + Some(json!([{"role": "user", "content": [ + {"type": "input_text", "text": "What is the capital of France?"}, + {"type": "input_text", "text": "Answer briefly."}, + {"type": "input_image", "image_url": "https://example.com/paris.png"}, + ]}])), + Some("What is the capital of France?\nAnswer briefly."), +)] +#[case::model_objects_after_dump( None, - Some(r#"[{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "hi", "annotations": []}]}]"#), - Some(r#"hi"#), + Some(json!([ + {"content": [{"text": "model dump prompt"}]}, + {"content": [{"output_text": "dict prompt"}]}, + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + ])), + Some("model dump prompt\ndict prompt\ninline prompt"), )] #[case::object_content( None, - Some(r#"{"content": [{"text": "object content prompt"}]}"#), - Some(r#"object content prompt"#) + Some(json!({"content": [{"text": "object content prompt"}]})), + Some("object content prompt"), )] -#[case::string_content(None, Some(r#"{"content": " inline "}"#), Some(r#"inline"#))] -#[case::unknown_keys_kept( +#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))] +#[case::null_content_uses_text_keys( None, - Some(r#"{"text": " ", "input_text": "fallback prompt"}"#), - Some(r#"{"text":" ","input_text":"fallback prompt"}"#) + Some(json!({"content": null, "output": "tool output"})), + Some("tool output"), )] -#[case::nested_lists( +#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)] +#[case::text_key_precedence( None, - Some(r#"[["a", [" b "]], "", "c"]"#), - Some( - r#"a -b -c"# - ) -)] -#[case::scalars_kept( + Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})), + Some("a"), +)] +#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))] +#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))] +#[case::non_string_text_keys_skipped( None, - Some(r#"[1, true, null, "kept"]"#), - Some( - r#"1 -true -kept"# - ) -)] -#[case::blank_text_parts(None, Some(r#"[{"type": "input_text", "text": " "}]"#), None)] + Some(json!({"text": 1, "output": "fallback"})), + Some("fallback"), +)] +#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] +#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] #[case::responses_function_call( None, - Some(r#"[{"role": "user", "content": "update the config"}, {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}]"#), - Some(r#"update the config -{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\"path\":\"a.yaml\"}"} -{"type":"function_call_output","call_id":1,"output":"ok"}"#), + Some(json!([ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ])), + Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), )] #[case::responses_structured_function_call_output( None, - Some(r#"[{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": " denied "}]}]"#), - Some(r#"{"type":"function_call","call_id":1,"name":"write_file","arguments":"{}"} -{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":" denied "}]}"#), + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, + ])), + Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), )] -#[case::responses_parallel_outputs_tagged_with_their_call( +#[case::responses_multi_part_output_joined_by_lines( None, - Some(r#"[{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}]"#), - Some(r#"{"type":"function_call","call_id":1,"name":"read","arguments":"a"} -{"type":"function_call","call_id":2,"name":"read","arguments":"b"} -{"type":"function_call_output","call_id":2,"output":"B"} -{"type":"function_call_output","call_id":1,"output":"A"}"#), + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [ + {"type": "input_text", "text": " line one "}, + {"type": "input_text", "text": "line two"}, + ]}, + ])), + Some(r#"{"name":"run","arguments":"{}"} +{"result_of_call":1,"output":"line one\nline two"}"#), )] -#[case::responses_reasoning_digested( +#[case::responses_parallel_outputs_tagged_with_their_call( None, - Some(r#"[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAAB"}]"#), - Some(r#"{"type":"reasoning","id":1,"encrypted_content":"sha256:010b9eb723d8d1e2"}"#), + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, + {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, + {"type": "function_call_output", "call_id": "c2", "output": "B"}, + {"type": "function_call_output", "call_id": "c1", "output": "A"}, + ])), + Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), +)] +#[case::responses_unknown_call_id_encodes_a_null_position( + None, + Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), + Some(r#"{"result_of_call":null,"output":"orphan"}"#), )] fn prompt_from_context_matches_python( - #[case] messages: Option<&str>, - #[case] input: Option<&str>, + #[case] messages: Option, + #[case] input: Option, #[case] expected: Option<&str>, ) { - let parse = |value: &str| serde_json::from_str::(value).unwrap(); - let context = context(messages.map(parse), input.map(parse)); - assert_eq!(prompt_from_context(&context).as_deref(), expected); + assert_eq!( + prompt_from_context(&context(messages, input)).as_deref(), + expected + ); +} + +/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank +/// text key falls through to the next one. +#[rstest] +#[case::blank_text_falls_through( + json!({"text": " ", "input_text": "fallback prompt"}), + "fallback prompt", +)] +fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) { + assert_eq!( + prompt_from_context(&context(None, Some(input))).as_deref(), + Some(expected) + ); +} + +/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from +/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and +/// strip also removes the ASCII information separators. +#[rstest] +#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)] +#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")] +#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")] +#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")] +#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")] +#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")] +fn python_serialization_edge_cases( + #[case] messages: Value, + #[case] input: Option, + #[case] expected: &str, +) { + let actual = match input { + Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(), + None => str_from_messages(messages.as_array().unwrap()), + }; + assert_eq!(actual, expected); } #[rstest] diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 9bf889899588..827fa91398fe 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,7 +22,9 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages_with_tools, - get_str_from_responses_input, + tool_call_ordinals, + tool_call_str, + tool_result_str, ) from litellm.types.utils import EmbeddingResponse @@ -269,7 +271,101 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: if "input" not in kwargs: return None - return get_str_from_responses_input(kwargs.get("input")) + responses_input: Final = kwargs.get("input") + prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) + return prompt or None + + @classmethod + def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: + prompt_parts: Final[list[str]] = [] + cls._collect_responses_input_text(value, prompt_parts, call_ordinals) + return "\n".join(prompt_parts).strip() + + @classmethod + def _collect_responses_input_text( + cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: + value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) + if value is None: + return + + if isinstance(value, str): + stripped_value: Final = value.strip() + if stripped_value: + prompt_parts.append(stripped_value) + return + + if isinstance(value, (list, tuple)): + for item in value: + cls._collect_responses_input_text(item, prompt_parts, call_ordinals) + return + + if isinstance(value, dict): + content = value.get("content") + if content is not None: + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) + return + + cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) + return + + content = getattr(value, "content", None) + if content is not None: + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) + return + + for text_key in ("text", "output", "input_text", "output_text"): + text_value = getattr(value, text_key, None) + if isinstance(text_value, str): + stripped_text = text_value.strip() + if stripped_text: + prompt_parts.append(stripped_text) + return + + @classmethod + def _collect_responses_text_fields( + cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: + for text_key in ("text", "output", "input_text", "output_text"): + text_value = value.get(text_key) + if isinstance(text_value, (list, tuple)): + cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) + return + if isinstance(text_value, str) and (stripped_text := text_value.strip()): + prompt_parts.append(stripped_text) + return + + @classmethod + def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: + items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () + dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) + return tool_call_ordinals( + item.get("call_id") + for item in dumped_items + if isinstance(item, dict) and item.get("type") == "function_call" + ) + + @classmethod + def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: + if not isinstance(value, dict): + return value + if value.get("type") == "function_call": + return tool_call_str(value.get("name"), value.get("arguments")) + if value.get("type") != "function_call_output": + return value + return tool_result_str( + value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) + ) + + @staticmethod + def _coerce_response_input_value(value: object) -> object: + model_dump: Final = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump() + dict_method: Final = getattr(value, "dict", None) + if callable(dict_method): + return dict_method() + return value def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8e75ac4034eb..0c52fae86d79 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,7 +2,6 @@ Common utility functions used for translating messages across providers """ -import hashlib import io import json import mimetypes @@ -196,148 +195,93 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text -_PLAIN_TYPES: Final = frozenset({"text", "input_text", "output_text", "message"}) -_PLAIN_KEYS: Final = frozenset({"role", "type", "text", "content", "status"}) -_TOOL_ROLES: Final = frozenset({"tool", "function"}) -_IGNORED_KEYS: Final = frozenset({"cache_control"}) -_CALL_ID_KEYS: Final = frozenset({"id", "call_id", "tool_use_id", "tool_call_id"}) -_OPAQUE_KEYS: Final = frozenset({"data", "file_data", "signature", "encrypted_content"}) - - def get_str_from_messages_with_tools(messages: object) -> str: """ - Semantic-cache prompt for chat-style ``messages``. Plain text parts stay plain text so text-only prompts embed - as they always have. Every other part (tool calls and results, images, unknown block types, extra fields) is kept - as compact JSON, so a new request shape changes the prompt instead of silently vanishing from it + ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns + that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) + produce different text. Each result is encoded with the position of the call it answers, since call ids are + random per session """ message_mappings: Final = tuple(_str_mappings(messages)) - call_ordinals: Final = _call_id_ordinals(message_mappings) - return "".join(_message_prompt(message, call_ordinals) for message in message_mappings) - - -def get_str_from_responses_input(responses_input: object) -> str | None: - """``get_str_from_messages_with_tools`` for Responses API ``input``, one stripped line per part""" - call_ordinals: Final = _call_id_ordinals(responses_input) - prompt: Final = "\n".join(_responses_input_parts(responses_input, call_ordinals)).strip() - return prompt or None + call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) + return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) -def _message_prompt(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - role: Final = message.get("role") - if (isinstance(role, str) and role in _TOOL_ROLES) or not _is_plain(message): - return _compact_json(_normalized(message, call_ordinals)) - return _plain_prompt(message, call_ordinals) +def tool_call_str(name: object, arguments: object) -> str: + return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' -def _plain_prompt(value: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - text: Final = value.get("text") - return (text if isinstance(text, str) else "") + _content_prompt(value.get("content"), call_ordinals) +def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: + ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None + return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' -def _content_prompt(content: object, call_ordinals: Mapping[str, int]) -> str: - value: Final = _plain_value(content) - if value is None: - return "" - if isinstance(value, str): - return value - if isinstance(value, (list, tuple)): - return "".join(_content_prompt(item, call_ordinals) for item in value) - if _is_str_mapping(value) and _is_plain(value): - return _plain_prompt(value, call_ordinals) - return _compact_json(_normalized(value, call_ordinals)) +def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: + string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) + return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) -def _responses_input_parts(value: object, call_ordinals: Mapping[str, int]) -> Iterator[str]: - item: Final = _plain_value(value) - if item is None: - return - if isinstance(item, str): - yield from _stripped(item) - elif isinstance(item, (list, tuple)): - for child in item: - yield from _responses_input_parts(child, call_ordinals) - elif _is_str_mapping(item) and _is_plain(item): - text: Final = item.get("text") - yield from _stripped(text if isinstance(text, str) else "") - yield from _responses_input_parts(item.get("content"), call_ordinals) - else: - yield _compact_json(_normalized(item, call_ordinals)) +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), default=str) -def _stripped(text: str) -> Iterator[str]: - stripped: Final = text.strip() - if stripped: - yield stripped +def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: + for message in messages: + yield from ( + block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" + ) + yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) -def _is_plain(value: Mapping[str, object]) -> bool: - value_type: Final = value.get("type") - text: Final = value.get("text") +def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + content: Final = _content_str_with_tools(message.get("content"), call_ordinals) return ( - (value_type is None or (isinstance(value_type, str) and value_type in _PLAIN_TYPES)) - and (text is None or isinstance(text, str)) - and all(key in _PLAIN_KEYS or key in _CALL_ID_KEYS for key in _kept_keys(value)) - ) - - -def _kept_keys(value: Mapping[str, object]) -> Iterator[str]: - return (key for key, item in value.items() if key not in _IGNORED_KEYS and not _is_empty(item)) - - -def _normalized(value: object, call_ordinals: Mapping[str, int], key: str | None = None) -> object: - item: Final = _plain_value(value) - if _is_str_mapping(item): - return MappingProxyType({kept: _normalized(item[kept], call_ordinals, kept) for kept in _kept_keys(item)}) - if isinstance(item, (list, tuple)): - return tuple(_normalized(child, call_ordinals) for child in item) - if not isinstance(item, str): - return item - if key in _CALL_ID_KEYS: - return call_ordinals.get(item, item) - if key in _OPAQUE_KEYS or item.startswith("data:"): - return f"sha256:{hashlib.sha256(item.encode('utf-8', 'surrogatepass')).hexdigest()[:16]}" - return item - - -def _call_id_ordinals(value: object) -> Mapping[str, int]: - return MappingProxyType( - {call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(_call_ids(value)), start=1)} + ( + tool_result_str(message.get("tool_call_id"), call_ordinals, content) + if message.get("role") == "tool" + else content + ) + + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) + + extract_search_results_text(message.get("search_results")) ) -def _call_ids(value: object, key: str | None = None) -> Iterator[str]: - item: Final = _plain_value(value) - if isinstance(item, str): - if key in _CALL_ID_KEYS: - yield item - elif _is_str_mapping(item): - for child_key, child in item.items(): - if child_key not in _IGNORED_KEYS: - yield from _call_ids(child, child_key) - elif isinstance(item, (list, tuple)): - for child in item: - yield from _call_ids(child) - - -def _is_empty(value: object) -> bool: - return value is None or (isinstance(value, (str, list, tuple, Mapping)) and not value) - - -def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=_json_default) +def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: + if isinstance(content, str): + return content + return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) -def _json_default(value: object) -> object: - return dict(value) if isinstance(value, MappingProxyType) else str(value) +def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + block_type: Final = block.get("type") + if block_type == "tool_use": + return tool_call_str(block.get("name"), block.get("input")) + if block_type == "tool_result": + return tool_result_str( + block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) + ) + text: Final = block.get("text") + return text if isinstance(text, str) else "" -def _plain_value(value: object) -> object: - return value.model_dump() if isinstance(value, BaseModel) else value +def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: + function: Final = _as_str_mapping(tool_call.get("function")) + if function is None: + return tool_call_str(None, None) + return tool_call_str(function.get("name"), function.get("arguments")) def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: items: Final = values if isinstance(values, (list, tuple)) else () - return (mapping for item in items if _is_str_mapping(mapping := _plain_value(item))) + return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) + + +def _as_str_mapping(value: object) -> Mapping[str, object] | None: + if isinstance(value, BaseModel): + return value.model_dump() + if _is_str_mapping(value): + return value + return None def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 6040ae2b214c..3359796ec84d 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -531,7 +531,7 @@ def test_redis_semantic_cache_get_cache_uses_responses_string_input(): ) -def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_responses_input(): +def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) @@ -561,7 +561,7 @@ def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_response ) redis_semantic_cache.llmcache.store.assert_called_once_with( - 'What is the capital of France?\nAnswer briefly.\n{"type":"input_image","image_url":"https://example.com/paris.png"}', + "What is the capital of France?\nAnswer briefly.", "{'content': 'Paris'}", vector=[0.1, 0.2, 0.3], filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, @@ -593,11 +593,10 @@ def turn(command: str) -> list[dict[str, object]]: ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}' - '{"type":"tool_result","tool_use_id":1,"content":"ok"}' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' ) - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) != ( - RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( + 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' ) @@ -613,8 +612,7 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ) assert prompt == ( - 'update the config\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}' - '\n{"type":"function_call_output","call_id":1,"output":"ok"}' + 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' ) @@ -634,13 +632,30 @@ def prompt_for(output_text: str) -> str | None: ] ) - assert prompt_for("wrote 5 bytes") == ( - 'write hello\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n' - '{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":"wrote 5 bytes"}]}' + expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' + assert ( + prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' ) assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") +def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], + }, + ] + ) + + assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' + + def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -655,63 +670,80 @@ def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | N ) assert prompt_for("c2", "c1") == ( - '{"type":"function_call","call_id":1,"name":"read","arguments":"a"}\n' - '{"type":"function_call","call_id":2,"name":"read","arguments":"b"}\n' - '{"type":"function_call_output","call_id":2,"output":"empty"}\n' - '{"type":"function_call_output","call_id":1,"output":"secret"}' + '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' + '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' ) assert prompt_for("c1", "c2") != prompt_for("c2", "c1") -def test_redis_semantic_cache_prompt_extraction_replaces_encrypted_reasoning_with_a_digest(): +def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - def prompt_for(encrypted_content: str) -> str | None: - return RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content}] - ) + class ModelDumpInput: + def model_dump(self): + return {"content": [{"text": "model dump prompt"}]} - blob = "gAAAAB" * 100 - prompt = prompt_for(blob) - assert prompt is not None - assert blob not in prompt - assert prompt.startswith('{"type":"reasoning","id":1,"encrypted_content":"sha256:') - assert prompt != prompt_for(blob + "x") + class DictInput: + def dict(self): + return {"content": [{"output_text": "dict prompt"}]} + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + ModelDumpInput(), + DictInput(), + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + ] + ) -def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): - from pydantic import BaseModel + assert prompt == "model dump prompt\ndict prompt\ninline prompt" + +def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ResponseInputMessage(BaseModel): - content: list[dict[str, str]] + assert RedisSemanticCache._get_prompt_from_kwargs() is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None + assert ( + RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "input_image", "image_url": "https://example.com"}] + ) + is None + ) + + +def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - ResponseInputMessage(content=[{"type": "input_text", "text": "model dump prompt"}]), - {"content": [{"type": "output_text", "text": "inline prompt", "annotations": []}]}, - ] + input={"text": " ", "input_text": "fallback prompt"} ) - assert prompt == "model dump prompt\ninline prompt" + assert prompt == "fallback prompt" -def test_redis_semantic_cache_prompt_extraction_keeps_image_only_input(): +def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - assert RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "input_image", "image_url": "https://example.com/a.png"}] - ) == '{"type":"input_image","image_url":"https://example.com/a.png"}' + class ResponseInput: + text = " " + input_text = "fallback prompt" + + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + assert prompt == "fallback prompt" -def test_redis_semantic_cache_prompt_extraction_returns_none_without_input(): + +def test_redis_semantic_cache_prompt_extraction_handles_object_content(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - assert RedisSemanticCache._get_prompt_from_kwargs() is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_text", "text": " "}]) is None + class ResponseInput: + content = [{"text": "object content prompt"}] + + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + + assert prompt == "object content prompt" def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 468bc71f95dd..59663de68d05 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -3,7 +3,6 @@ import json import os import sys -from collections.abc import Callable from typing import Final import pytest @@ -2024,12 +2023,12 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], }, ], - 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}', - id="anthropic-tool-use-with-its-call-position", + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', + id="anthropic-tool-use-name-and-input-without-id", ), pytest.param( [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - 'fix the failing test{"type":"tool_result","tool_use_id":1,"content":"calc.py"}', + 'fix the failing test{"result_of_call":null,"output":"calc.py"}', id="anthropic-string-tool-result", ), pytest.param( @@ -2046,7 +2045,7 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ], } ], - '{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next', + '{"result_of_call":null,"output":"ab"}next', id="anthropic-nested-text-tool-result-then-text", ), pytest.param( @@ -2062,11 +2061,9 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], - 'fix the failing test{"role":"assistant","content":"writing","tool_calls":[' - '{"id":1,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}},' - '{"id":2,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}}]}' - '{"role":"tool","tool_call_id":1,"content":"ok"}', - id="openai-tool-calls-then-tool-result", + 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' + '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', + id="openai-tool-calls-in-order-before-tool-result", ), pytest.param( [ @@ -2085,9 +2082,8 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ], }, ], - '{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}' - '{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}' - '{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}', + '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' + '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', id="anthropic-parallel-tool-results-tagged-with-their-call", ), pytest.param( @@ -2097,11 +2093,19 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "content": None, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], }, - {"role": "tool", "tool_call_id": "call_0", "content": "C"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, ], - '{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}' - '{"role":"tool","tool_call_id":1,"content":"C"}', - id="null-content-dropped-from-encoded-message", + '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' + '{"result_of_call":2,"output":"C"}', + id="reused-call-ids-keep-first-position", ), pytest.param( [ @@ -2112,13 +2116,13 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "search_results": [{"source": "s", "title": "t"}], } ], - '{"role":"tool","tool_call_id":1,"content":"small","search_results":[{"source":"s","title":"t"}]}', - id="tool-result-keeps-search-results", + '{"result_of_call":null,"output":"small"}st', + id="tool-result-search-results-follow-the-encoded-result", ), pytest.param( - [{"role": "tool", "content": 'x"}{"role":"tool","content":"y'}], - '{"role":"tool","content":"x\\"}{\\"role\\":\\"tool\\",\\"content\\":\\"y"}', - id="tool-output-is-escaped-even-without-a-call-id", + [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], + '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', + id="tool-output-cannot-forge-an-encoded-result", ), pytest.param( [ @@ -2127,32 +2131,17 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], ) ], - '{"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"read"},"id":1,"type":"function"}]}', + '{"name":"read","arguments":"{}"}', id="openai-response-message-object", ), pytest.param( - [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], - "hi", - id="cache-control-ignored", - ), - pytest.param( - [{"role": "user", "name": "alice", "content": "hi"}], - '{"role":"user","name":"alice","content":"hi"}', - id="extra-message-field-kept", - ), - pytest.param( - [{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}], - '{"type":"brand_new_block","payload":{"k":"v"}}!', - id="unknown-block-type-kept", - ), - pytest.param( - [{"role": "user", "content": [{"type": "text", "text": 3}]}], - '{"type":"text","text":3}', - id="non-string-text-kept", + [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], + '{"name":null,"arguments":null}', + id="malformed-tool-call-entries", ), ], ) -def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object], expected: str) -> None: +def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: assert get_str_from_messages_with_tools(messages) == expected @@ -2161,54 +2150,25 @@ def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object [ pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), pytest.param( - [{"role": "user", "content": [{"type": "text", "text": "what is "}, {"type": "text", "text": "this"}]}], - id="text-parts", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this"}, + ], + } + ], + id="text-and-image-parts", ), pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_str_from_messages_with_tools_matches_get_str_from_messages_for_text(messages: list[object]) -> None: +def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures -def _anthropic_image(data: str) -> list[object]: - return [{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "data": data}}]}] - - -def _openai_image(url: str) -> list[object]: - return [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}] - - -def _thinking(signature: str) -> list[object]: - return [{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": signature}]}] - - -@pytest.mark.parametrize( - ("build", "blob"), - [ - pytest.param(_anthropic_image, "iVBORw0KGgo" * 50, id="anthropic-base64-image"), - pytest.param(_openai_image, "data:image/png;base64," + "iVBORw0KGgo" * 50, id="openai-data-url-image"), - pytest.param(_thinking, "EqQBCkgIARABGAIiQL" * 20, id="thinking-signature"), - ], -) -def test_get_str_from_messages_with_tools_replaces_opaque_blobs_with_distinct_digests( - build: Callable[[str], list[object]], blob: str -) -> None: - prompt: Final = get_str_from_messages_with_tools(build(blob)) - - assert blob not in prompt - assert "sha256:" in prompt - assert len(prompt) < len(blob) - assert prompt == get_str_from_messages_with_tools(build(blob)) - assert prompt != get_str_from_messages_with_tools(build(blob + "A")) - - -def test_get_str_from_messages_with_tools_keeps_image_urls_distinct() -> None: - assert get_str_from_messages_with_tools(_openai_image("https://example.com/a.png")) != ( - get_str_from_messages_with_tools(_openai_image("https://example.com/b.png")) - ) - - def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: return [ { From b9a794291a93cd6f4c14a05776e13aaeae7c2c3e Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 00:24:48 +0000 Subject: [PATCH 15/29] refactor(caching): rename get_str_from_messages_with_tools to get_semantic_cache_prompt_from_messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 4 +-- litellm/caching/qdrant_semantic_cache.py | 10 +++---- litellm/caching/redis_semantic_cache.py | 4 +-- .../prompt_templates/common_utils.py | 2 +- ...ore_utils_prompt_templates_common_utils.py | 26 ++++++++++--------- 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 253efb477596..5af6a54d8947 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -1,6 +1,6 @@ //! The embedding and prompt contract every semantic backend shares. //! -//! Python's semantic caches all read their prompt through `get_str_from_messages_with_tools`, and +//! Python's semantic caches all read their prompt through `get_semantic_cache_prompt_from_messages`, and //! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API //! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. @@ -83,7 +83,7 @@ impl Embedder for PreparedEmbedding { } } -/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, +/// `get_semantic_cache_prompt_from_messages`: every message's content text, tool calls and tool results, /// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the /// position of the call it answers. pub fn str_from_messages(messages: &[Value]) -> String { diff --git a/litellm/caching/qdrant_semantic_cache.py b/litellm/caching/qdrant_semantic_cache.py index 8fe03131760e..1258c9f4917f 100644 --- a/litellm/caching/qdrant_semantic_cache.py +++ b/litellm/caching/qdrant_semantic_cache.py @@ -24,7 +24,7 @@ ) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages_with_tools, + get_semantic_cache_prompt_from_messages, ) from litellm.types.utils import EmbeddingResponse @@ -286,7 +286,7 @@ def set_cache(self, key, value, **kwargs): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages_with_tools(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) # create an embedding for prompt embedding_response: Final = cast( @@ -325,7 +325,7 @@ def get_cache(self, key, **kwargs): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages_with_tools(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) # convert to embedding embedding_response: Final = cast( @@ -400,7 +400,7 @@ async def async_set_cache(self, key, value, **kwargs): # get the prompt messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages_with_tools(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) # get the embedding @@ -435,7 +435,7 @@ async def async_get_cache(self, key, **kwargs): # get the messages messages: Final = kwargs["messages"] - prompt: Final = get_str_from_messages_with_tools(messages) + prompt: Final = get_semantic_cache_prompt_from_messages(messages) embedding_response: Final = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata")) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 827fa91398fe..dfbdcce8ae81 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -21,7 +21,7 @@ from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( - get_str_from_messages_with_tools, + get_semantic_cache_prompt_from_messages, tool_call_ordinals, tool_call_str, tool_result_str, @@ -266,7 +266,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: """ messages: Final = kwargs.get("messages") if messages: - return get_str_from_messages_with_tools(messages) + return get_semantic_cache_prompt_from_messages(messages) if "input" not in kwargs: return None diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index a7769e6b1ac1..f191286144f8 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -195,7 +195,7 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text -def get_str_from_messages_with_tools(messages: object) -> str: +def get_semantic_cache_prompt_from_messages(messages: object) -> str: """ ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 826bd83bc542..61d7cdbd96c0 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -16,8 +16,8 @@ encrypted_reasoning_signature, get_file_ids_from_messages, get_format_from_file_id, + get_semantic_cache_prompt_from_messages, get_str_from_messages, - get_str_from_messages_with_tools, handle_any_messages_to_chat_completion_str_messages_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, @@ -2308,8 +2308,8 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s ), ], ) -def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: - assert get_str_from_messages_with_tools(messages) == expected +def test_get_semantic_cache_prompt_from_messages_keeps_tool_exchange(messages: list[object], expected: str) -> None: + assert get_semantic_cache_prompt_from_messages(messages) == expected @pytest.mark.parametrize( @@ -2332,8 +2332,10 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: - assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures +def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_without_tools( + messages: list[object], +) -> None: + assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: @@ -2356,13 +2358,13 @@ def _results_in_swapped_order(result_for_a: str, result_for_b: str) -> list[obje return [call, answer_b, answer_a] -def test_get_str_from_messages_with_tools_tells_apart_parallel_results_answering_different_calls() -> None: - assert get_str_from_messages_with_tools(_parallel_reads("empty", "secret")) != get_str_from_messages_with_tools( - _results_in_swapped_order("secret", "empty") - ) +def test_get_semantic_cache_prompt_from_messages_tells_apart_parallel_results_answering_different_calls() -> None: + assert get_semantic_cache_prompt_from_messages( + _parallel_reads("empty", "secret") + ) != get_semantic_cache_prompt_from_messages(_results_in_swapped_order("secret", "empty")) -def test_get_str_from_messages_with_tools_ignores_call_ids_that_differ_between_sessions() -> None: - assert get_str_from_messages_with_tools( +def test_get_semantic_cache_prompt_from_messages_ignores_call_ids_that_differ_between_sessions() -> None: + assert get_semantic_cache_prompt_from_messages( _parallel_reads("A", "B", call_id_prefix="toolu_") - ) == get_str_from_messages_with_tools(_parallel_reads("A", "B")) + ) == get_semantic_cache_prompt_from_messages(_parallel_reads("A", "B")) From b27722ce584d23623a27ba0560c97ffcf67e3c26 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 00:41:13 +0000 Subject: [PATCH 16/29] refactor(caching): split semantic cache prompt extraction by API format Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 10 +- litellm/caching/redis_semantic_cache.py | 100 +-------- .../prompt_templates/common_utils.py | 194 ++++++++++++------ 3 files changed, 139 insertions(+), 165 deletions(-) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 5af6a54d8947..26274fd43248 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -1,8 +1,9 @@ //! The embedding and prompt contract every semantic backend shares. //! //! Python's semantic caches all read their prompt through `get_semantic_cache_prompt_from_messages`, and -//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API -//! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. +//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API `input` +//! through `get_semantic_cache_prompt_from_responses_input`. Qdrant reads messages only. Each +//! backend picks one of the two extractors here. use std::{collections::HashMap, future::Future, io}; @@ -207,8 +208,9 @@ pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { (!messages.is_empty()).then(|| str_from_messages(messages)) } -/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a -/// Responses API `input`. `None` when neither yields a prompt. +/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then +/// `get_semantic_cache_prompt_from_responses_input` over a Responses API `input`. `None` when +/// neither yields a prompt. pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) && !messages.is_empty() diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index dfbdcce8ae81..342dc77f88dc 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,9 +22,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_semantic_cache_prompt_from_messages, - tool_call_ordinals, - tool_call_str, - tool_result_str, + get_semantic_cache_prompt_from_responses_input, ) from litellm.types.utils import EmbeddingResponse @@ -271,101 +269,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: if "input" not in kwargs: return None - responses_input: Final = kwargs.get("input") - prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) - return prompt or None - - @classmethod - def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: - prompt_parts: Final[list[str]] = [] - cls._collect_responses_input_text(value, prompt_parts, call_ordinals) - return "\n".join(prompt_parts).strip() - - @classmethod - def _collect_responses_input_text( - cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) - if value is None: - return - - if isinstance(value, str): - stripped_value: Final = value.strip() - if stripped_value: - prompt_parts.append(stripped_value) - return - - if isinstance(value, (list, tuple)): - for item in value: - cls._collect_responses_input_text(item, prompt_parts, call_ordinals) - return - - if isinstance(value, dict): - content = value.get("content") - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) - return - - content = getattr(value, "content", None) - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - for text_key in ("text", "output", "input_text", "output_text"): - text_value = getattr(value, text_key, None) - if isinstance(text_value, str): - stripped_text = text_value.strip() - if stripped_text: - prompt_parts.append(stripped_text) - return - - @classmethod - def _collect_responses_text_fields( - cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - for text_key in ("text", "output", "input_text", "output_text"): - text_value = value.get(text_key) - if isinstance(text_value, (list, tuple)): - cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) - return - if isinstance(text_value, str) and (stripped_text := text_value.strip()): - prompt_parts.append(stripped_text) - return - - @classmethod - def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: - items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () - dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) - return tool_call_ordinals( - item.get("call_id") - for item in dumped_items - if isinstance(item, dict) and item.get("type") == "function_call" - ) - - @classmethod - def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: - if not isinstance(value, dict): - return value - if value.get("type") == "function_call": - return tool_call_str(value.get("name"), value.get("arguments")) - if value.get("type") != "function_call_output": - return value - return tool_result_str( - value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) - ) - - @staticmethod - def _coerce_response_input_value(value: object) -> object: - model_dump: Final = getattr(value, "model_dump", None) - if callable(model_dump): - return model_dump() - dict_method: Final = getattr(value, "dict", None) - if callable(dict_method): - return dict_method() - return value + return get_semantic_cache_prompt_from_responses_input(kwargs.get("input")) or None def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index f191286144f8..59c582e181e6 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,7 +13,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from pydantic import BaseModel from typing_extensions import TypeIs # noqa: TID251 # narrows untyped message payloads without a runtime conversion import litellm @@ -197,94 +196,163 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: def get_semantic_cache_prompt_from_messages(messages: object) -> str: """ - ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns - that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) - produce different text. Each result is encoded with the position of the call it answers, since call ids are - random per session + The text the semantic cache embeds for ``messages``, shared by the Messages API and Chat Completions. Keeps + the text ``get_str_from_messages`` keeps, plus every tool call and tool result, so agent turns that differ + only in their tool exchange embed differently. Call ids are random per session, so each result names the + position of the call it answers instead """ - message_mappings: Final = tuple(_str_mappings(messages)) - call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) - return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) + message_dicts: Final = _dumped_dicts(messages) + positions: Final = _tool_call_positions(_messages_tool_call_ids(message_dicts)) + return "".join(_message_text(message, positions) for message in message_dicts) -def tool_call_str(name: object, arguments: object) -> str: - return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' +def get_semantic_cache_prompt_from_responses_input(responses_input: object) -> str: + """ + The text the semantic cache embeds for a Responses API ``input``: each text part stripped and on its own + line, with ``function_call`` and ``function_call_output`` items encoded like tool calls and results above + """ + items: Final = tuple(_dumped(item) for item in _sequence(responses_input)) + call_ids: Final = ( + item.get("call_id") for item in items if isinstance(item, Mapping) and item.get("type") == "function_call" + ) + return _responses_text(responses_input, _tool_call_positions(call_ids)) -def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: - ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None - return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' +def _message_text(message: Mapping[str, object], positions: Mapping[str, int]) -> str: + content_text: Final = _content_text(message.get("content"), positions) + return _chat_completions_message_text(message, content_text, positions) + extract_search_results_text( + message.get("search_results") + ) -def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: - string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) - return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) +def _content_text(content: object, positions: Mapping[str, int]) -> str: + if isinstance(content, str): + return content + return "".join(_messages_api_block_text(block, positions) for block in _dumped_dicts(content)) -def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=str) +def _messages_api_block_text(block: Mapping[str, object], positions: Mapping[str, int]) -> str: + if block.get("type") == "tool_use": + return _tool_call_text(block.get("name"), block.get("input")) + if block.get("type") == "tool_result": + return _tool_result_text(block.get("tool_use_id"), _content_text(block.get("content"), positions), positions) + text: Final = block.get("text") + return text if isinstance(text, str) else "" -def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: +def _chat_completions_message_text( + message: Mapping[str, object], content_text: str, positions: Mapping[str, int] +) -> str: + result_or_content: Final = ( + _tool_result_text(message.get("tool_call_id"), content_text, positions) + if message.get("role") == "tool" + else content_text + ) + return result_or_content + "".join( + _chat_completions_tool_call_text(tool_call) for tool_call in _dumped_dicts(message.get("tool_calls")) + ) + + +def _chat_completions_tool_call_text(tool_call: Mapping[str, object]) -> str: + function: Final = _dumped(tool_call.get("function")) + if not isinstance(function, Mapping): + return _tool_call_text(None, None) + return _tool_call_text(function.get("name"), function.get("arguments")) + + +def _messages_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: for message in messages: - yield from ( - block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" - ) - yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) + for block in _dumped_dicts(message.get("content")): + if block.get("type") == "tool_use": + yield block.get("id") + for tool_call in _dumped_dicts(message.get("tool_calls")): + yield tool_call.get("id") -def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - content: Final = _content_str_with_tools(message.get("content"), call_ordinals) - return ( - ( - tool_result_str(message.get("tool_call_id"), call_ordinals, content) - if message.get("role") == "tool" - else content - ) - + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) - + extract_search_results_text(message.get("search_results")) - ) +def _responses_text(value: object, positions: Mapping[str, int]) -> str: + return "\n".join(_responses_text_parts(value, positions)).strip() -def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: - if isinstance(content, str): - return content - return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) +def _responses_text_parts(value: object, positions: Mapping[str, int]) -> Iterator[str]: + item: Final = _dumped(value) + if item is None: + return + if isinstance(item, str): + if item.strip(): + yield item.strip() + return + if isinstance(item, (list, tuple)): + for nested in item: + yield from _responses_text_parts(nested, positions) + return + if isinstance(item, Mapping) and item.get("type") == "function_call": + yield _tool_call_text(item.get("name"), item.get("arguments")) + return + if isinstance(item, Mapping) and item.get("type") == "function_call_output": + yield _tool_result_text(item.get("call_id"), _responses_text(item.get("output"), positions), positions) + return + content: Final = _field(item, "content") + if content is not None: + yield from _responses_text_parts(content, positions) + return + yield from _responses_first_text_field(item, positions) -def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - block_type: Final = block.get("type") - if block_type == "tool_use": - return tool_call_str(block.get("name"), block.get("input")) - if block_type == "tool_result": - return tool_result_str( - block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) - ) - text: Final = block.get("text") - return text if isinstance(text, str) else "" +def _responses_first_text_field(item: object, positions: Mapping[str, int]) -> Iterator[str]: + for key in ("text", "output", "input_text", "output_text"): + text: Final = _field(item, key) + if isinstance(text, (list, tuple)): + yield from _responses_text_parts(text, positions) + return + if isinstance(text, str) and text.strip(): + yield text.strip() + return -def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: - function: Final = _as_str_mapping(tool_call.get("function")) - if function is None: - return tool_call_str(None, None) - return tool_call_str(function.get("name"), function.get("arguments")) +def _tool_call_text(name: object, arguments: object) -> str: + return _compact_json({"name": name, "arguments": arguments}) -def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: - items: Final = values if isinstance(values, (list, tuple)) else () - return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) +def _tool_result_text(call_id: object, output: str, positions: Mapping[str, int]) -> str: + position: Final = positions.get(call_id) if isinstance(call_id, str) else None + return _compact_json({"result_of_call": position, "output": output}) -def _as_str_mapping(value: object) -> Mapping[str, object] | None: - if isinstance(value, BaseModel): - return value.model_dump() - if _is_str_mapping(value): - return value - return None +def _tool_call_positions(call_ids: Iterable[object]) -> Mapping[str, int]: + string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) + return MappingProxyType({call_id: position for position, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), default=str) + + +def _dumped_dicts(values: object) -> tuple[Mapping[str, object], ...]: + return tuple(item for item in map(_dumped, _sequence(values)) if _is_str_mapping(item)) + + +def _sequence(values: object) -> Sequence[object]: + return values if isinstance(values, (list, tuple)) else () + + +def _dumped(value: object) -> object: + """pydantic models, and anything else that dumps itself, as the dict the request carried""" + model_dump: Final = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump() + dict_method: Final = getattr(value, "dict", None) + if callable(dict_method): + return dict_method() + return value + + +def _field(item: object, key: str) -> object: + if isinstance(item, Mapping): + return item.get(key) + return getattr(item, key, None) -def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str +def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: return isinstance(value, Mapping) From 7c1390c8364126e4e3c1f08643bc9570851d98a2 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 00:54:14 +0000 Subject: [PATCH 17/29] fix(caching): drop TypeIs guard and register Responses prompt walker with the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 8 +------- tests/code_coverage_tests/recursive_detector.py | 14 ++++---------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 59c582e181e6..3e5aa4747cf6 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,8 +13,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from typing_extensions import TypeIs # noqa: TID251 # narrows untyped message payloads without a runtime conversion - import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -328,7 +326,7 @@ def _compact_json(value: object) -> str: def _dumped_dicts(values: object) -> tuple[Mapping[str, object], ...]: - return tuple(item for item in map(_dumped, _sequence(values)) if _is_str_mapping(item)) + return tuple(item for item in map(_dumped, _sequence(values)) if isinstance(item, Mapping)) def _sequence(values: object) -> Sequence[object]: @@ -352,10 +350,6 @@ def _field(item: object, key: str) -> object: return getattr(item, key, None) -def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: - return isinstance(value, Mapping) - - def is_non_content_values_set(message: AllMessageValues) -> bool: ignore_keys: Final = ["content", "role", "name"] return any(message.get(key, None) is not None for key in message if key not in ignore_keys) diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 863e8befcf9a..4a024f6f7cbf 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -23,6 +23,7 @@ "_can_object_call_model", # max depth set. "encode_unserializable_types", # max depth set. "filter_value_from_dict", # max depth set. + "_responses_text_parts", # walks only the nesting a Responses `input` carries; same walk the RedisSemanticCache classmethod did before it moved here. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. "_remove_json_schema_refs", # max depth set., @@ -106,13 +107,8 @@ def _is_recursive_call(self, func_node, call_node): return True # Case 2: Method call with self (e.g., self.my_func()) - if isinstance(call_node.func, ast.Attribute) and isinstance( - call_node.func.value, ast.Name - ): - return ( - call_node.func.value.id == "self" - and call_node.func.attr == func_node.name - ) + if isinstance(call_node.func, ast.Attribute) and isinstance(call_node.func.value, ast.Name): + return call_node.func.value.id == "self" and call_node.func.attr == func_node.name return False @@ -147,9 +143,7 @@ def find_recursive_functions_in_directory(directory): # this is used in the CI/CD pipeline to prevent recursive functions from being merged directory_path = "./litellm" - recursive_functions, ignored_recursive_functions = ( - find_recursive_functions_in_directory(directory_path) - ) + recursive_functions, ignored_recursive_functions = find_recursive_functions_in_directory(directory_path) print("UNIGNORED RECURSIVE FUNCTIONS: ", recursive_functions) print("IGNORED RECURSIVE FUNCTIONS: ", ignored_recursive_functions) From 1c4a69bfbe34c6c7d4b716837abe50dc8d341fe3 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 01:17:36 +0000 Subject: [PATCH 18/29] fix(caching): pick the Responses text field without a Final inside a loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 3e5aa4747cf6..683e02e9e8fd 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -297,14 +297,16 @@ def _responses_text_parts(value: object, positions: Mapping[str, int]) -> Iterat def _responses_first_text_field(item: object, positions: Mapping[str, int]) -> Iterator[str]: - for key in ("text", "output", "input_text", "output_text"): - text: Final = _field(item, key) - if isinstance(text, (list, tuple)): - yield from _responses_text_parts(text, positions) - return - if isinstance(text, str) and text.strip(): - yield text.strip() - return + fields: Final = (_field(item, key) for key in ("text", "output", "input_text", "output_text")) + text: Final = next((field for field in fields if _is_text_field(field)), None) + if isinstance(text, (list, tuple)): + yield from _responses_text_parts(text, positions) + elif isinstance(text, str): + yield text.strip() + + +def _is_text_field(field: object) -> bool: + return isinstance(field, (list, tuple)) or (isinstance(field, str) and bool(field.strip())) def _tool_call_text(name: object, arguments: object) -> str: From 4ac923b6e7bd7516a03d21187080b221482bee16 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 18:06:38 +0000 Subject: [PATCH 19/29] refactor(caching): walk semantic cache prompts as plain dicts, dumping pydantic items once up front Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 108 +++++++----------- .../code_coverage_tests/recursive_detector.py | 3 +- .../unit/caching/test_redis_semantic_cache.py | 105 ++++------------- 3 files changed, 67 insertions(+), 149 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 683e02e9e8fd..ae611c44664d 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,6 +13,8 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast +from pydantic import BaseModel + import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -199,7 +201,7 @@ def get_semantic_cache_prompt_from_messages(messages: object) -> str: only in their tool exchange embed differently. Call ids are random per session, so each result names the position of the call it answers instead """ - message_dicts: Final = _dumped_dicts(messages) + message_dicts: Final = _dicts(_plain(messages)) positions: Final = _tool_call_positions(_messages_tool_call_ids(message_dicts)) return "".join(_message_text(message, positions) for message in message_dicts) @@ -209,11 +211,9 @@ def get_semantic_cache_prompt_from_responses_input(responses_input: object) -> s The text the semantic cache embeds for a Responses API ``input``: each text part stripped and on its own line, with ``function_call`` and ``function_call_output`` items encoded like tool calls and results above """ - items: Final = tuple(_dumped(item) for item in _sequence(responses_input)) - call_ids: Final = ( - item.get("call_id") for item in items if isinstance(item, Mapping) and item.get("type") == "function_call" - ) - return _responses_text(responses_input, _tool_call_positions(call_ids)) + items: Final = _plain(responses_input) + call_ids: Final = (item.get("call_id") for item in _dicts(items) if item.get("type") == "function_call") + return _responses_text(items, _tool_call_positions(call_ids)) def _message_text(message: Mapping[str, object], positions: Mapping[str, int]) -> str: @@ -226,7 +226,7 @@ def _message_text(message: Mapping[str, object], positions: Mapping[str, int]) - def _content_text(content: object, positions: Mapping[str, int]) -> str: if isinstance(content, str): return content - return "".join(_messages_api_block_text(block, positions) for block in _dumped_dicts(content)) + return "".join(_messages_api_block_text(block, positions) for block in _dicts(content)) def _messages_api_block_text(block: Mapping[str, object], positions: Mapping[str, int]) -> str: @@ -247,12 +247,12 @@ def _chat_completions_message_text( else content_text ) return result_or_content + "".join( - _chat_completions_tool_call_text(tool_call) for tool_call in _dumped_dicts(message.get("tool_calls")) + _chat_completions_tool_call_text(tool_call) for tool_call in _dicts(message.get("tool_calls")) ) def _chat_completions_tool_call_text(tool_call: Mapping[str, object]) -> str: - function: Final = _dumped(tool_call.get("function")) + function: Final = tool_call.get("function") if not isinstance(function, Mapping): return _tool_call_text(None, None) return _tool_call_text(function.get("name"), function.get("arguments")) @@ -260,53 +260,37 @@ def _chat_completions_tool_call_text(tool_call: Mapping[str, object]) -> str: def _messages_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: for message in messages: - for block in _dumped_dicts(message.get("content")): + for block in _dicts(message.get("content")): if block.get("type") == "tool_use": yield block.get("id") - for tool_call in _dumped_dicts(message.get("tool_calls")): + for tool_call in _dicts(message.get("tool_calls")): yield tool_call.get("id") def _responses_text(value: object, positions: Mapping[str, int]) -> str: - return "\n".join(_responses_text_parts(value, positions)).strip() - - -def _responses_text_parts(value: object, positions: Mapping[str, int]) -> Iterator[str]: - item: Final = _dumped(value) - if item is None: - return - if isinstance(item, str): - if item.strip(): - yield item.strip() - return - if isinstance(item, (list, tuple)): - for nested in item: - yield from _responses_text_parts(nested, positions) - return - if isinstance(item, Mapping) and item.get("type") == "function_call": - yield _tool_call_text(item.get("name"), item.get("arguments")) - return - if isinstance(item, Mapping) and item.get("type") == "function_call_output": - yield _tool_result_text(item.get("call_id"), _responses_text(item.get("output"), positions), positions) - return - content: Final = _field(item, "content") - if content is not None: - yield from _responses_text_parts(content, positions) - return - yield from _responses_first_text_field(item, positions) + return "\n".join(_responses_text_lines(value, positions)).strip() -def _responses_first_text_field(item: object, positions: Mapping[str, int]) -> Iterator[str]: - fields: Final = (_field(item, key) for key in ("text", "output", "input_text", "output_text")) - text: Final = next((field for field in fields if _is_text_field(field)), None) - if isinstance(text, (list, tuple)): - yield from _responses_text_parts(text, positions) - elif isinstance(text, str): - yield text.strip() +def _responses_text_lines(value: object, positions: Mapping[str, int]) -> Iterator[str]: + if isinstance(value, str): + if value.strip(): + yield value.strip() + elif isinstance(value, list): + for item in value: + yield from _responses_text_lines(item, positions) + elif isinstance(value, Mapping): + yield from _responses_item_lines(value, positions) -def _is_text_field(field: object) -> bool: - return isinstance(field, (list, tuple)) or (isinstance(field, str) and bool(field.strip())) +def _responses_item_lines(item: Mapping[str, object], positions: Mapping[str, int]) -> Iterator[str]: + if item.get("type") == "function_call": + yield _tool_call_text(item.get("name"), item.get("arguments")) + elif item.get("type") == "function_call_output": + yield _tool_result_text(item.get("call_id"), _responses_text(item.get("output"), positions), positions) + elif item.get("content") is not None: + yield from _responses_text_lines(item.get("content"), positions) + else: + yield from _responses_text_lines(item.get("text"), positions) def _tool_call_text(name: object, arguments: object) -> str: @@ -327,29 +311,21 @@ def _compact_json(value: object) -> str: return json.dumps(value, separators=(",", ":"), default=str) -def _dumped_dicts(values: object) -> tuple[Mapping[str, object], ...]: - return tuple(item for item in map(_dumped, _sequence(values)) if isinstance(item, Mapping)) - - -def _sequence(values: object) -> Sequence[object]: - return values if isinstance(values, (list, tuple)) else () - - -def _dumped(value: object) -> object: - """pydantic models, and anything else that dumps itself, as the dict the request carried""" - model_dump: Final = getattr(value, "model_dump", None) - if callable(model_dump): - return model_dump() - dict_method: Final = getattr(value, "dict", None) - if callable(dict_method): - return dict_method() +def _plain(value: object) -> object: + """SDK callers pass pydantic items (``input += response.output``); dump them to the dicts the request carried""" + if isinstance(value, BaseModel): + return value.model_dump() + if isinstance(value, (list, tuple)): + return [_plain(item) for item in value] + if isinstance(value, Mapping): + return {key: _plain(item) for key, item in value.items()} return value -def _field(item: object, key: str) -> object: - if isinstance(item, Mapping): - return item.get(key) - return getattr(item, key, None) +def _dicts(values: object) -> tuple[Mapping[str, object], ...]: + if not isinstance(values, list): + return () + return tuple(value for value in values if isinstance(value, Mapping)) def is_non_content_values_set(message: AllMessageValues) -> bool: diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 4a024f6f7cbf..457f9fb62341 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -23,7 +23,8 @@ "_can_object_call_model", # max depth set. "encode_unserializable_types", # max depth set. "filter_value_from_dict", # max depth set. - "_responses_text_parts", # walks only the nesting a Responses `input` carries; same walk the RedisSemanticCache classmethod did before it moved here. + "_responses_text_lines", # walks only the nesting a Responses `input` carries. + "_plain", # walks only the nesting a request `messages` or `input` carries. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. "_remove_json_schema_refs", # max depth set., diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 3359796ec84d..07ec324699b7 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -203,10 +203,7 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): assert redis_semantic_cache.llmcache is fallback_cache_mock assert semantic_cache_mock.call_args_list[0].kwargs["name"] == "existing_index" - assert ( - semantic_cache_mock.call_args_list[1].kwargs["name"] - == "existing_index_isolated" - ) + assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index_isolated" assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ RedisSemanticCache._cache_key_filterable_field() ] @@ -236,10 +233,7 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) assert redis_semantic_cache.llmcache is fallback_cache_mock - assert ( - semantic_cache_mock.call_args_list[2].kwargs["name"] - == "existing_index_isolated" - ) + assert semantic_cache_mock.call_args_list[2].kwargs["name"] == "existing_index_isolated" assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ RedisSemanticCache._cache_key_filterable_field() @@ -363,9 +357,7 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): ] redis_semantic_cache.llmcache.acheck = AsyncMock(return_value=mock_result) - redis_semantic_cache._get_async_embedding = AsyncMock( - return_value=[0.1, 0.2, 0.3] - ) + redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) with patch.object( redis_semantic_cache, @@ -413,9 +405,7 @@ async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeyp } ] ) - redis_semantic_cache._get_async_embedding = AsyncMock( - return_value=[0.1, 0.2, 0.3] - ) + redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) with patch.object( redis_semantic_cache, @@ -447,9 +437,7 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) redis_semantic_cache.llmcache.astore = AsyncMock() - redis_semantic_cache._get_async_embedding = AsyncMock( - return_value=[0.1, 0.2, 0.3] - ) + redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) await redis_semantic_cache.async_set_cache( key="test_key", @@ -676,27 +664,25 @@ def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | N assert prompt_for("c1", "c2") != prompt_for("c2", "c1") -def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - class ModelDumpInput: - def model_dump(self): - return {"content": [{"text": "model dump prompt"}]} +def test_redis_semantic_cache_prompt_extraction_dumps_sdk_response_items_appended_to_input(): + from openai.types.responses import ResponseFunctionToolCall - class DictInput: - def dict(self): - return {"content": [{"output_text": "dict prompt"}]} + from litellm.caching.redis_semantic_cache import RedisSemanticCache prompt = RedisSemanticCache._get_prompt_from_kwargs( input=[ - ModelDumpInput(), - DictInput(), - {"content": [{"input_text": "inline prompt"}]}, - {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + {"role": "user", "content": "write hello"}, + ResponseFunctionToolCall( + type="function_call", call_id="c1", name="write_file", arguments='{"path":"a.txt"}' + ), + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, ] ) - assert prompt == "model dump prompt\ndict prompt\ninline prompt" + assert ( + prompt + == 'write hello\n{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n{"result_of_call":1,"output":"ok"}' + ) def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): @@ -706,46 +692,11 @@ def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None assert ( - RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "input_image", "image_url": "https://example.com"}] - ) + RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_image", "image_url": "https://example.com"}]) is None ) -def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input={"text": " ", "input_text": "fallback prompt"} - ) - - assert prompt == "fallback prompt" - - -def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - class ResponseInput: - text = " " - input_text = "fallback prompt" - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) - - assert prompt == "fallback prompt" - - -def test_redis_semantic_cache_prompt_extraction_handles_object_content(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - class ResponseInput: - content = [{"text": "object content prompt"}] - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) - - assert prompt == "object content prompt" - - def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -964,9 +915,7 @@ def test_redis_get_embedding_falls_back_to_direct(monkeypatch): fake_proxy.llm_model_list = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_proxy) - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]} - ) as direct_embed: + with patch("litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]}) as direct_embed: vec = cache._get_embedding("hello") assert vec == [0.1, 0.2] @@ -1102,9 +1051,7 @@ def test_redis_sync_set_cache_passes_precomputed_vector(): cache = RedisSemanticCache.__new__(RedisSemanticCache) cache.llmcache = MagicMock() - cache._get_cache_filters = MagicMock( - return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"} - ) + cache._get_cache_filters = MagicMock(return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}) cache._get_ttl = MagicMock(return_value=None) cache._get_embedding = MagicMock(return_value=[0.1, 0.2, 0.3]) @@ -1141,9 +1088,7 @@ def test_redis_sync_get_cache_passes_precomputed_vector(): ) cache._get_embedding = MagicMock(return_value=[0.1, 0.2, 0.3]) - with patch.object( - cache, "_get_cache_key_filter_expression", return_value="cache-key-filter" - ): + with patch.object(cache, "_get_cache_key_filter_expression", return_value="cache-key-filter"): result = cache.get_cache( key="test_key", messages=[{"content": "What is the capital of France?"}], @@ -1257,9 +1202,7 @@ def test_redis_get_embedding_truncates_direct_path_with_explicit_limit(monkeypat fake_proxy.llm_model_list = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_proxy) - with patch( - "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]} - ) as direct_embed: + with patch("litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]}) as direct_embed: cache._get_embedding(LONG_PROMPT) sent_input = direct_embed.call_args.kwargs["input"] @@ -1306,9 +1249,7 @@ def test_redis_init_defers_redisvl_construction(monkeypatch): def test_redis_failed_llmcache_build_is_not_memoized(monkeypatch): built_cache = MagicMock() - semantic_cache_mock = MagicMock( - side_effect=[ConnectionError("redis down"), built_cache] - ) + semantic_cache_mock = MagicMock(side_effect=[ConnectionError("redis down"), built_cache]) custom_vectorizer_mock = MagicMock() with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): From ffd3a85a1811b6e7f8420e06b306bc4b2aa436ba Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 18:18:12 +0000 Subject: [PATCH 20/29] refactor(caching): write the semantic cache prompt builders as plain loops Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 153 ++++++++---------- .../code_coverage_tests/recursive_detector.py | 3 +- 2 files changed, 66 insertions(+), 90 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index ae611c44664d..6f8499669023 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -201,64 +201,24 @@ def get_semantic_cache_prompt_from_messages(messages: object) -> str: only in their tool exchange embed differently. Call ids are random per session, so each result names the position of the call it answers instead """ - message_dicts: Final = _dicts(_plain(messages)) - positions: Final = _tool_call_positions(_messages_tool_call_ids(message_dicts)) - return "".join(_message_text(message, positions) for message in message_dicts) - - -def get_semantic_cache_prompt_from_responses_input(responses_input: object) -> str: - """ - The text the semantic cache embeds for a Responses API ``input``: each text part stripped and on its own - line, with ``function_call`` and ``function_call_output`` items encoded like tool calls and results above - """ - items: Final = _plain(responses_input) - call_ids: Final = (item.get("call_id") for item in _dicts(items) if item.get("type") == "function_call") - return _responses_text(items, _tool_call_positions(call_ids)) - - -def _message_text(message: Mapping[str, object], positions: Mapping[str, int]) -> str: - content_text: Final = _content_text(message.get("content"), positions) - return _chat_completions_message_text(message, content_text, positions) + extract_search_results_text( - message.get("search_results") - ) - - -def _content_text(content: object, positions: Mapping[str, int]) -> str: - if isinstance(content, str): - return content - return "".join(_messages_api_block_text(block, positions) for block in _dicts(content)) - - -def _messages_api_block_text(block: Mapping[str, object], positions: Mapping[str, int]) -> str: - if block.get("type") == "tool_use": - return _tool_call_text(block.get("name"), block.get("input")) - if block.get("type") == "tool_result": - return _tool_result_text(block.get("tool_use_id"), _content_text(block.get("content"), positions), positions) - text: Final = block.get("text") - return text if isinstance(text, str) else "" - - -def _chat_completions_message_text( - message: Mapping[str, object], content_text: str, positions: Mapping[str, int] -) -> str: - result_or_content: Final = ( - _tool_result_text(message.get("tool_call_id"), content_text, positions) - if message.get("role") == "tool" - else content_text - ) - return result_or_content + "".join( - _chat_completions_tool_call_text(tool_call) for tool_call in _dicts(message.get("tool_calls")) - ) - - -def _chat_completions_tool_call_text(tool_call: Mapping[str, object]) -> str: - function: Final = tool_call.get("function") - if not isinstance(function, Mapping): - return _tool_call_text(None, None) - return _tool_call_text(function.get("name"), function.get("arguments")) + message_dicts = _dicts(_plain(messages)) + positions = _tool_call_positions(_messages_tool_call_ids(message_dicts)) + parts = [] + for message in message_dicts: + content = message.get("content") + text = content if isinstance(content, str) else _messages_api_blocks_text(content, positions) + if message.get("role") == "tool": + text = _tool_result_text(message.get("tool_call_id"), text, positions) + for tool_call in _dicts(message.get("tool_calls")): + function = tool_call.get("function") + if not isinstance(function, Mapping): + function = {} + text += _tool_call_text(function.get("name"), function.get("arguments")) + parts.append(text + extract_search_results_text(message.get("search_results"))) + return "".join(parts) -def _messages_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: +def _messages_tool_call_ids(messages: Sequence[Mapping[str, object]]) -> Iterator[object]: for message in messages: for block in _dicts(message.get("content")): if block.get("type") == "tool_use": @@ -267,48 +227,63 @@ def _messages_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterato yield tool_call.get("id") -def _responses_text(value: object, positions: Mapping[str, int]) -> str: - return "\n".join(_responses_text_lines(value, positions)).strip() +def _messages_api_blocks_text(blocks: object, positions: Mapping[str, int]) -> str: + text = "" + for block in _dicts(blocks): + if block.get("type") == "tool_use": + text += _tool_call_text(block.get("name"), block.get("input")) + elif block.get("type") == "tool_result": + content = block.get("content") + output = content if isinstance(content, str) else _messages_api_blocks_text(content, positions) + text += _tool_result_text(block.get("tool_use_id"), output, positions) + elif isinstance(block.get("text"), str): + text += str(block["text"]) + return text -def _responses_text_lines(value: object, positions: Mapping[str, int]) -> Iterator[str]: +def get_semantic_cache_prompt_from_responses_input(responses_input: object) -> str: + """ + The text the semantic cache embeds for a Responses API ``input``: each text part stripped and on its own + line, with ``function_call`` and ``function_call_output`` items encoded like tool calls and results above + """ + items = _plain(responses_input) + call_ids = [item.get("call_id") for item in _dicts(items) if item.get("type") == "function_call"] + return _responses_text(items, _tool_call_positions(call_ids)) + + +def _responses_text(value: object, positions: Mapping[str, int]) -> str: if isinstance(value, str): - if value.strip(): - yield value.strip() - elif isinstance(value, list): - for item in value: - yield from _responses_text_lines(item, positions) - elif isinstance(value, Mapping): - yield from _responses_item_lines(value, positions) - - -def _responses_item_lines(item: Mapping[str, object], positions: Mapping[str, int]) -> Iterator[str]: - if item.get("type") == "function_call": - yield _tool_call_text(item.get("name"), item.get("arguments")) - elif item.get("type") == "function_call_output": - yield _tool_result_text(item.get("call_id"), _responses_text(item.get("output"), positions), positions) - elif item.get("content") is not None: - yield from _responses_text_lines(item.get("content"), positions) - else: - yield from _responses_text_lines(item.get("text"), positions) + return value.strip() + if isinstance(value, list): + lines = [_responses_text(item, positions) for item in value] + return "\n".join(line for line in lines if line) + if not isinstance(value, Mapping): + return "" + if value.get("type") == "function_call": + return _tool_call_text(value.get("name"), value.get("arguments")) + if value.get("type") == "function_call_output": + output = _responses_text(value.get("output"), positions) + return _tool_result_text(value.get("call_id"), output, positions) + if value.get("content") is not None: + return _responses_text(value.get("content"), positions) + return _responses_text(value.get("text"), positions) def _tool_call_text(name: object, arguments: object) -> str: - return _compact_json({"name": name, "arguments": arguments}) + return json.dumps({"name": name, "arguments": arguments}, separators=(",", ":"), default=str) def _tool_result_text(call_id: object, output: str, positions: Mapping[str, int]) -> str: - position: Final = positions.get(call_id) if isinstance(call_id, str) else None - return _compact_json({"result_of_call": position, "output": output}) + position = positions.get(call_id) if isinstance(call_id, str) else None + return json.dumps({"result_of_call": position, "output": output}, separators=(",", ":"), default=str) def _tool_call_positions(call_ids: Iterable[object]) -> Mapping[str, int]: - string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) - return MappingProxyType({call_id: position for position, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) - - -def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=str) + positions = {} + for call_id in call_ids: + if isinstance(call_id, str) and call_id not in positions: + positions[call_id] = len(positions) + 1 + return positions def _plain(value: object) -> object: @@ -322,10 +297,10 @@ def _plain(value: object) -> object: return value -def _dicts(values: object) -> tuple[Mapping[str, object], ...]: +def _dicts(values: object) -> list[Mapping[str, object]]: if not isinstance(values, list): - return () - return tuple(value for value in values if isinstance(value, Mapping)) + return [] + return [value for value in values if isinstance(value, Mapping)] def is_non_content_values_set(message: AllMessageValues) -> bool: diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 457f9fb62341..21af1a651924 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -23,7 +23,8 @@ "_can_object_call_model", # max depth set. "encode_unserializable_types", # max depth set. "filter_value_from_dict", # max depth set. - "_responses_text_lines", # walks only the nesting a Responses `input` carries. + "_responses_text", # walks only the nesting a Responses `input` carries. + "_messages_api_blocks_text", # walks only the nesting a `tool_result` block carries. "_plain", # walks only the nesting a request `messages` or `input` carries. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. From 394abff55a93c668dfe85510b871a83adda45836 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 23:22:19 +0000 Subject: [PATCH 21/29] fix(caching): skip the cache past max_messages and keep tool_result text in semantic prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cache-qdrant-semantic/tests/qdrant.rs | 2 +- litellm-rust/crates/cache/src/semantic.rs | 184 ++---- litellm-rust/crates/cache/tests/semantic.rs | 171 ++---- litellm/caching/caching.py | 17 + litellm/caching/redis_semantic_cache.py | 61 +- .../prompt_templates/common_utils.py | 129 +---- .../code_coverage_tests/recursive_detector.py | 16 +- .../caching/test_cache_max_messages.py | 415 +++++++++++++ .../caching/test_semantic_cache_tool_turns.py | 544 ------------------ tests/unit/caching/test_caching.py | 65 ++- .../unit/caching/test_redis_semantic_cache.py | 148 ++--- ...ore_utils_prompt_templates_common_utils.py | 203 ++----- 12 files changed, 771 insertions(+), 1184 deletions(-) create mode 100644 tests/integration/caching/test_cache_max_messages.py delete mode 100644 tests/integration/caching/test_semantic_cache_tool_turns.py diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index 7a27519c2691..fe25c503a368 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# + r#"sourcetitlebody{"page":1,"section":"intro"}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 26274fd43248..5aafb333b391 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -1,15 +1,14 @@ //! The embedding and prompt contract every semantic backend shares. //! -//! Python's semantic caches all read their prompt through `get_semantic_cache_prompt_from_messages`, and -//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API `input` -//! through `get_semantic_cache_prompt_from_responses_input`. Qdrant reads messages only. Each -//! backend picks one of the two extractors here. +//! Python's semantic caches all read their prompt through `get_str_from_messages`, and +//! `RedisSemanticCache._get_prompt_from_kwargs` (inherited by Valkey) adds Responses API +//! `input`. Qdrant reads messages only. Each backend picks one of the two extractors here. -use std::{collections::HashMap, future::Future, io}; +use std::{future::Future, io}; use serde::Serialize; use serde_json::{ - Value, json, + Value, ser::{CharEscape, Formatter, Serializer}, }; @@ -84,122 +83,45 @@ impl Embedder for PreparedEmbedding { } } -/// `get_semantic_cache_prompt_from_messages`: every message's content text, tool calls and tool results, -/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the -/// position of the call it answers. +/// `get_semantic_cache_prompt_from_messages`: every message's text content, including the text of +/// Messages API `tool_result` blocks, followed by its search results. pub fn str_from_messages(messages: &[Value]) -> String { - let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); - let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { - let block_ids = message - .get("content") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) - .filter_map(|block| block.get("id")); - let tool_call_ids = message - .get("tool_calls") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|tool_call| tool_call.get("id")); - block_ids.chain(tool_call_ids) - })); let mut text = String::new(); - for message in messages { - if message.get("role").and_then(Value::as_str) == Some("tool") { - let mut output = String::new(); - push_content_text(&mut output, message.get("content"), &call_ordinals); - text.push_str(&tool_result_json( - message.get("tool_call_id"), - &call_ordinals, - &output, - )); - } else { - push_content_text(&mut text, message.get("content"), &call_ordinals); - } - if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { - for tool_call in tool_calls.iter().filter_map(Value::as_object) { - let function = tool_call.get("function"); - text.push_str(&tool_call_json( - function.and_then(|function| function.get("name")), - function.and_then(|function| function.get("arguments")), - )); + for message in messages.iter().filter_map(Value::as_object) { + match message.get("content") { + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(blocks)) => { + for block in blocks { + push_block_text(&mut text, block); + } } + _ => {} } push_search_results_text(&mut text, message.get("search_results")); } text } -/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. -fn push_content_text( - text: &mut String, - content: Option<&Value>, - call_ordinals: &HashMap<&str, usize>, -) { - match content { - Some(Value::String(content)) => text.push_str(content), +fn push_block_text(text: &mut String, block: &Value) { + if block.get("type").and_then(Value::as_str) != Some("tool_result") { + push_text_field(text, block); + return; + } + match block.get("content") { + Some(Value::String(result)) => text.push_str(result), Some(Value::Array(blocks)) => { - for block in blocks.iter().filter_map(Value::as_object) { - match block.get("type").and_then(Value::as_str) { - Some("tool_use") => { - text.push_str(&tool_call_json(block.get("name"), block.get("input"))); - } - Some("tool_result") => { - let mut output = String::new(); - push_content_text(&mut output, block.get("content"), call_ordinals); - text.push_str(&tool_result_json( - block.get("tool_use_id"), - call_ordinals, - &output, - )); - } - _ => { - if let Some(block_text) = block.get("text").and_then(Value::as_str) { - text.push_str(block_text); - } - } - } + for inner in blocks { + push_text_field(text, inner); } } _ => {} } } -/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. -fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { - let mut ordinals = HashMap::new(); - for call_id in call_ids.filter_map(Value::as_str) { - let next = ordinals.len() + 1; - ordinals.entry(call_id).or_insert(next); +fn push_text_field(text: &mut String, block: &Value) { + if let Some(block_text) = block.get("text").and_then(Value::as_str) { + text.push_str(block_text); } - ordinals -} - -/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result -/// answers no known call. -fn tool_result_json( - call_id: Option<&Value>, - call_ordinals: &HashMap<&str, usize>, - output: &str, -) -> String { - let ordinal = call_id - .and_then(Value::as_str) - .and_then(|call_id| call_ordinals.get(call_id)); - format!( - "{{\"result_of_call\":{},\"output\":{}}}", - compact_json(&json!(ordinal)), - compact_json(&Value::String(output.to_owned())), - ) -} - -/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. -fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { - compact_json(&json!({ - "name": name.unwrap_or(&Value::Null), - "arguments": arguments.unwrap_or(&Value::Null), - })) } /// The messages prompt Qdrant embeds: `None` when the request carries no messages. @@ -208,9 +130,8 @@ pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { (!messages.is_empty()).then(|| str_from_messages(messages)) } -/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then -/// `get_semantic_cache_prompt_from_responses_input` over a Responses API `input`. `None` when -/// neither yields a prompt. +/// `RedisSemanticCache._get_prompt_from_kwargs`: chat messages first, then the text parts of a +/// Responses API `input`. `None` when neither yields a prompt. pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { if let Some(messages) = context.messages.as_ref().and_then(Value::as_array) && !messages.is_empty() @@ -218,16 +139,8 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { return Some(str_from_messages(messages)); } let input = context.input.as_ref()?; - let call_ordinals = tool_call_ordinals( - input - .as_array() - .into_iter() - .flatten() - .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) - .filter_map(|item| item.get("call_id")), - ); let mut parts = Vec::new(); - collect_input_text(input, &mut parts, &call_ordinals); + collect_input_text(input, &mut parts); let prompt = python_strip(&parts.join("\n")).to_owned(); (!prompt.is_empty()).then_some(prompt) } @@ -256,49 +169,26 @@ fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { } } -fn collect_input_text( - value: &Value, - parts: &mut Vec, - call_ordinals: &HashMap<&str, usize>, -) { +fn collect_input_text(value: &Value, parts: &mut Vec) { match value { Value::String(text) => { push_trimmed(text, parts); } Value::Array(items) => { for item in items { - collect_input_text(item, parts, call_ordinals); + collect_input_text(item, parts); } } Value::Object(map) => { - if map.get("type").and_then(Value::as_str) == Some("function_call") { - parts.push(tool_call_json(map.get("name"), map.get("arguments"))); - return; - } - if map.get("type").and_then(Value::as_str) == Some("function_call_output") { - let mut output_parts = Vec::new(); - if let Some(output) = map.get("output") { - collect_input_text(output, &mut output_parts, call_ordinals); - } - parts.push(tool_result_json( - map.get("call_id"), - call_ordinals, - python_strip(&output_parts.join("\n")), - )); - return; - } if let Some(content) = map.get("content").filter(|content| !content.is_null()) { - collect_input_text(content, parts, call_ordinals); + collect_input_text(content, parts); return; } for key in ["text", "output", "input_text", "output_text"] { - match map.get(key) { - Some(nested @ Value::Array(_)) => { - collect_input_text(nested, parts, call_ordinals); - return; - } - Some(Value::String(text)) if push_trimmed(text, parts) => return, - _ => {} + if let Some(Value::String(text)) = map.get(key) + && push_trimmed(text, parts) + { + return; } } } diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index af87e6e0164a..e0a39bb9d5a6 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -30,6 +30,31 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex ]}]), "What is this?", )] +#[case::tool_result_string( + json!([ + {"role": "user", "content": "list the files"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}, + ]}, + ]), + "list the filescalc.py test_calc.py", +)] +#[case::tool_result_blocks( + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [ + {"type": "text", "text": "x = 1"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + ]}, + ]}]), + "x = 1", +)] +#[case::tool_result_without_content( + json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}]), + "", +)] #[case::missing_null_and_empty_content( json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), "", @@ -38,143 +63,67 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex json!([{"role": "tool", "content": "small", "search_results": [ {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, ]}]), - r#"{"result_of_call":null,"output":"small"}sthidden payload"#, + "smallsthidden payload", )] #[case::title_only_search_result( json!([{"role": "tool", "content": "small", "search_results": [ {"source": "s", "title": "long title", "content": []}, ]}]), - r#"{"result_of_call":null,"output":"small"}slong title"#, + "smallslong title", )] #[case::search_results_without_content( json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), - r#"{"result_of_call":null,"output":""}st"#, + "st", )] #[case::search_result_fields_in_python_order( json!([{"role": "tool", "content": "c", "search_results": [ {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, {"source": "s2"}, ]}]), - r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, + r#"cstbody{"enabled":true}s2"#, )] #[case::null_citations_skipped( json!([{"role": "tool", "content": "c", "search_results": [ {"source": "s", "citations": null}, ]}]), - r#"{"result_of_call":null,"output":"c"}s"#, + "cs", )] #[case::non_string_and_non_object_entries_skipped( json!([{"role": "tool", "content": "c", "search_results": [ "junk", {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, ]}]), - r#"{"result_of_call":null,"output":"c"}kept"#, + "ckept", )] #[case::non_list_search_results_skipped( json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), - r#"{"result_of_call":null,"output":"c"}"#, + "c", )] #[case::citations_compact_in_insertion_order( json!([{"role": "tool", "search_results": [ {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, ]}]), - r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, + r#"{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, )] #[case::citations_ensure_ascii( json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), - r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, + r#"["caf\u00e9","\u4e2d"]"#, )] #[case::citations_astral_chars_as_surrogate_pairs( json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), - r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, + r#""\ud83d\ude00""#, )] #[case::citations_escapes( json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), - r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, + r#""q\"\\\n\t\u0001/""#, )] #[case::citations_large_float_exponent( json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), - r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, + "[1e+20,1.0]", )] #[case::citations_scalars( json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), - r#"{"result_of_call":null,"output":""}false3"#, -)] -#[case::anthropic_tool_use_name_and_input_without_id( - json!([ - {"role": "user", "content": "fix the failing test"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, - ]}, - ]), - r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, -)] -#[case::anthropic_string_tool_result( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, - ]}]), - r#"{"result_of_call":null,"output":"calc.py"}"#, -)] -#[case::anthropic_nested_text_tool_result_then_text( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": [ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ]}, - {"type": "text", "text": "next"}, - ]}]), - r#"{"result_of_call":null,"output":"ab"}next"#, -)] -#[case::openai_tool_calls_in_order_before_tool_result( - json!([ - {"role": "assistant", "content": "writing", "tool_calls": [ - {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, - {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, - ]}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"}, - ]), - r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, -)] -#[case::anthropic_parallel_tool_results_tagged_with_their_call( - json!([ - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, - {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, - ]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, - {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, - ]}, - ]), - r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, -)] -#[case::reused_call_ids_keep_first_position( - json!([ - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, - ]}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ]}, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, - ]), - r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, -)] -#[case::unknown_call_ids_encode_a_null_position( - json!([ - {"role": "tool", "tool_call_id": "c9", "content": "ok"}, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, - ]), - r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, -)] -#[case::tool_output_cannot_forge_an_encoded_result( - json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), - r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, -)] -#[case::malformed_tool_call_entries( - json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), - r#"{"name":null,"arguments":null}"#, + "false3", )] fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); @@ -268,50 +217,6 @@ fn prompt_from_messages_reads_messages_only( )] #[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] #[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] -#[case::responses_function_call( - None, - Some(json!([ - {"role": "user", "content": "update the config"}, - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, - {"type": "function_call_output", "call_id": "c1", "output": "ok"}, - ])), - Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), -)] -#[case::responses_structured_function_call_output( - None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, - ])), - Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), -)] -#[case::responses_multi_part_output_joined_by_lines( - None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [ - {"type": "input_text", "text": " line one "}, - {"type": "input_text", "text": "line two"}, - ]}, - ])), - Some(r#"{"name":"run","arguments":"{}"} -{"result_of_call":1,"output":"line one\nline two"}"#), -)] -#[case::responses_parallel_outputs_tagged_with_their_call( - None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, - {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, - {"type": "function_call_output", "call_id": "c2", "output": "B"}, - {"type": "function_call_output", "call_id": "c1", "output": "A"}, - ])), - Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), -)] -#[case::responses_unknown_call_id_encodes_a_null_position( - None, - Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), - Some(r#"{"result_of_call":null,"output":"orphan"}"#), -)] fn prompt_from_context_matches_python( #[case] messages: Option, #[case] input: Option, diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 4070f41fd32c..e6ad0ebdebe5 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -92,6 +92,17 @@ class CacheMode(str, Enum): #### LiteLLM.Completion / Embedding Cache #### +def _request_message_count(kwargs: Mapping[str, object]) -> int: + """Chat and Messages API `messages`, else Responses API `input` items; embedding `input` strings count as none""" + messages: Final = kwargs.get("messages") + if isinstance(messages, list): + return len(messages) + input_items: Final = kwargs.get("input") + if not isinstance(input_items, list): + return 0 + return sum(1 for item in input_items if isinstance(item, (Mapping, BaseModel))) + + class Cache: _native_cache: "ResponseCacheRuntime | None" = None @@ -142,6 +153,7 @@ def __init__( semantic_cache_embedding_max_input_tokens: int | None = None, semantic_cache_embedding_timeout: float | None = None, semantic_cache_scope: str = SemanticCacheScope.KEY.value, + max_messages: int | None = 4, # GCP IAM authentication parameters gcp_service_account: str | None = None, gcp_ssl_ca_certs: str | None = None, @@ -171,6 +183,7 @@ def __init__( semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens. semantic_cache_embedding_timeout (float, optional): Seconds a semantic-cache lookup may spend embedding the prompt before it gives up and lets the request continue to the LLM. Defaults to SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS. semantic_cache_scope (str, optional): "key" isolates semantic-cache buckets per key/team/org. "end_user" additionally isolates per end user (falls back to the key scope when the request carries no end-user id). Defaults to "key". + max_messages (int, optional): Requests with more `messages` (or Responses API `input` items) than this are neither looked up nor stored, so long agent conversations never serve or create a cache entry. None disables the limit. Defaults to 4. # Disk Cache Args disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None. @@ -321,6 +334,7 @@ def __init__( self.ttl = ttl self.mode: CacheMode = mode or CacheMode.default_on self.semantic_cache_scope: str = SemanticCacheScope(semantic_cache_scope).value + self.max_messages: int | None = max_messages if self.type == LiteLLMCacheType.LOCAL and default_in_memory_ttl is not None: self.ttl = default_in_memory_ttl @@ -1005,7 +1019,10 @@ def should_use_cache(self, **kwargs): If cache is default_on then this is True If cache is default_off then this is only true when user has opted in to use cache + Always False once the request carries more than `max_messages` messages """ + if self.max_messages is not None and _request_message_count(kwargs) > self.max_messages: + return False if self.mode == CacheMode.default_on: return True diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 342dc77f88dc..e39c2074739f 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,7 +22,6 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_semantic_cache_prompt_from_messages, - get_semantic_cache_prompt_from_responses_input, ) from litellm.types.utils import EmbeddingResponse @@ -269,7 +268,65 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: if "input" not in kwargs: return None - return get_semantic_cache_prompt_from_responses_input(kwargs.get("input")) or None + prompt_parts: Final[list[str]] = [] + cls._collect_responses_input_text(kwargs.get("input"), prompt_parts) + prompt: Final = "\n".join(prompt_parts).strip() + return prompt or None + + @classmethod + def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: + value = cls._coerce_response_input_value(value) + if value is None: + return + + if isinstance(value, str): + stripped_value: Final = value.strip() + if stripped_value: + prompt_parts.append(stripped_value) + return + + if isinstance(value, (list, tuple)): + for item in value: + cls._collect_responses_input_text(item, prompt_parts) + return + + if isinstance(value, dict): + content = value.get("content") + if content is not None: + cls._collect_responses_input_text(content, prompt_parts) + return + + for text_key in ("text", "output", "input_text", "output_text"): + text_value = value.get(text_key) + if isinstance(text_value, str): + stripped_text = text_value.strip() + if stripped_text: + prompt_parts.append(stripped_text) + return + return + + content = getattr(value, "content", None) + if content is not None: + cls._collect_responses_input_text(content, prompt_parts) + return + + for text_key in ("text", "output", "input_text", "output_text"): + text_value = getattr(value, text_key, None) + if isinstance(text_value, str): + stripped_text = text_value.strip() + if stripped_text: + prompt_parts.append(stripped_text) + return + + @staticmethod + def _coerce_response_input_value(value: object) -> object: + model_dump: Final = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump() + dict_method: Final = getattr(value, "dict", None) + if callable(dict_method): + return dict_method() + return value def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 6f8499669023..20dc2726de9e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -13,8 +13,6 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar, cast -from pydantic import BaseModel - import litellm from litellm import verbose_logger from litellm.router_utils.batch_utils import InMemoryFile @@ -194,113 +192,44 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text -def get_semantic_cache_prompt_from_messages(messages: object) -> str: +def get_semantic_cache_prompt_from_messages(messages: Sequence[Mapping[str, object]]) -> str: """ - The text the semantic cache embeds for ``messages``, shared by the Messages API and Chat Completions. Keeps - the text ``get_str_from_messages`` keeps, plus every tool call and tool result, so agent turns that differ - only in their tool exchange embed differently. Call ids are random per session, so each result names the - position of the call it answers instead + The text a semantic cache embeds for a request: `get_str_from_messages` plus the text inside + Messages API `tool_result` blocks, so a tool turn does not embed identically to the turn before it """ - message_dicts = _dicts(_plain(messages)) - positions = _tool_call_positions(_messages_tool_call_ids(message_dicts)) - parts = [] - for message in message_dicts: - content = message.get("content") - text = content if isinstance(content, str) else _messages_api_blocks_text(content, positions) - if message.get("role") == "tool": - text = _tool_result_text(message.get("tool_call_id"), text, positions) - for tool_call in _dicts(message.get("tool_calls")): - function = tool_call.get("function") - if not isinstance(function, Mapping): - function = {} - text += _tool_call_text(function.get("name"), function.get("arguments")) - parts.append(text + extract_search_results_text(message.get("search_results"))) - return "".join(parts) - - -def _messages_tool_call_ids(messages: Sequence[Mapping[str, object]]) -> Iterator[object]: - for message in messages: - for block in _dicts(message.get("content")): - if block.get("type") == "tool_use": - yield block.get("id") - for tool_call in _dicts(message.get("tool_calls")): - yield tool_call.get("id") + return "".join(_semantic_cache_message_text(message) for message in messages) -def _messages_api_blocks_text(blocks: object, positions: Mapping[str, int]) -> str: - text = "" - for block in _dicts(blocks): - if block.get("type") == "tool_use": - text += _tool_call_text(block.get("name"), block.get("input")) - elif block.get("type") == "tool_result": - content = block.get("content") - output = content if isinstance(content, str) else _messages_api_blocks_text(content, positions) - text += _tool_result_text(block.get("tool_use_id"), output, positions) - elif isinstance(block.get("text"), str): - text += str(block["text"]) - return text - - -def get_semantic_cache_prompt_from_responses_input(responses_input: object) -> str: - """ - The text the semantic cache embeds for a Responses API ``input``: each text part stripped and on its own - line, with ``function_call`` and ``function_call_output`` items encoded like tool calls and results above - """ - items = _plain(responses_input) - call_ids = [item.get("call_id") for item in _dicts(items) if item.get("type") == "function_call"] - return _responses_text(items, _tool_call_positions(call_ids)) +def _semantic_cache_message_text(message: Mapping[str, object]) -> str: + return _semantic_cache_content_text(message.get("content")) + extract_search_results_text( + message.get("search_results") + ) -def _responses_text(value: object, positions: Mapping[str, int]) -> str: - if isinstance(value, str): - return value.strip() - if isinstance(value, list): - lines = [_responses_text(item, positions) for item in value] - return "\n".join(line for line in lines if line) - if not isinstance(value, Mapping): +def _semantic_cache_content_text(content: object) -> str: + if isinstance(content, str): + return content + if not isinstance(content, list): return "" - if value.get("type") == "function_call": - return _tool_call_text(value.get("name"), value.get("arguments")) - if value.get("type") == "function_call_output": - output = _responses_text(value.get("output"), positions) - return _tool_result_text(value.get("call_id"), output, positions) - if value.get("content") is not None: - return _responses_text(value.get("content"), positions) - return _responses_text(value.get("text"), positions) + return "".join(_semantic_cache_block_text(block) for block in content) -def _tool_call_text(name: object, arguments: object) -> str: - return json.dumps({"name": name, "arguments": arguments}, separators=(",", ":"), default=str) - - -def _tool_result_text(call_id: object, output: str, positions: Mapping[str, int]) -> str: - position = positions.get(call_id) if isinstance(call_id, str) else None - return json.dumps({"result_of_call": position, "output": output}, separators=(",", ":"), default=str) - - -def _tool_call_positions(call_ids: Iterable[object]) -> Mapping[str, int]: - positions = {} - for call_id in call_ids: - if isinstance(call_id, str) and call_id not in positions: - positions[call_id] = len(positions) + 1 - return positions - - -def _plain(value: object) -> object: - """SDK callers pass pydantic items (``input += response.output``); dump them to the dicts the request carried""" - if isinstance(value, BaseModel): - return value.model_dump() - if isinstance(value, (list, tuple)): - return [_plain(item) for item in value] - if isinstance(value, Mapping): - return {key: _plain(item) for key, item in value.items()} - return value - - -def _dicts(values: object) -> list[Mapping[str, object]]: - if not isinstance(values, list): - return [] - return [value for value in values if isinstance(value, Mapping)] +def _semantic_cache_block_text(block: object) -> str: + if not isinstance(block, Mapping): + return "" + if block.get("type") != "tool_result": + return _text_field(block) + result: Final = block.get("content") + if isinstance(result, str): + return result + if isinstance(result, list): + return "".join(_text_field(inner) for inner in result) + return "" + + +def _text_field(block: object) -> str: + text: Final = block.get("text") if isinstance(block, Mapping) else None + return text if isinstance(text, str) else "" def is_non_content_values_set(message: AllMessageValues) -> bool: diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 21af1a651924..863e8befcf9a 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -23,9 +23,6 @@ "_can_object_call_model", # max depth set. "encode_unserializable_types", # max depth set. "filter_value_from_dict", # max depth set. - "_responses_text", # walks only the nesting a Responses `input` carries. - "_messages_api_blocks_text", # walks only the nesting a `tool_result` block carries. - "_plain", # walks only the nesting a request `messages` or `input` carries. "normalize_json_schema_types", # max depth set. "_extract_fields_recursive", # max depth set. "_remove_json_schema_refs", # max depth set., @@ -109,8 +106,13 @@ def _is_recursive_call(self, func_node, call_node): return True # Case 2: Method call with self (e.g., self.my_func()) - if isinstance(call_node.func, ast.Attribute) and isinstance(call_node.func.value, ast.Name): - return call_node.func.value.id == "self" and call_node.func.attr == func_node.name + if isinstance(call_node.func, ast.Attribute) and isinstance( + call_node.func.value, ast.Name + ): + return ( + call_node.func.value.id == "self" + and call_node.func.attr == func_node.name + ) return False @@ -145,7 +147,9 @@ def find_recursive_functions_in_directory(directory): # this is used in the CI/CD pipeline to prevent recursive functions from being merged directory_path = "./litellm" - recursive_functions, ignored_recursive_functions = find_recursive_functions_in_directory(directory_path) + recursive_functions, ignored_recursive_functions = ( + find_recursive_functions_in_directory(directory_path) + ) print("UNIGNORED RECURSIVE FUNCTIONS: ", recursive_functions) print("IGNORED RECURSIVE FUNCTIONS: ", ignored_recursive_functions) diff --git a/tests/integration/caching/test_cache_max_messages.py b/tests/integration/caching/test_cache_max_messages.py new file mode 100644 index 000000000000..33bd0c09b4c9 --- /dev/null +++ b/tests/integration/caching/test_cache_max_messages.py @@ -0,0 +1,415 @@ +import hashlib +import json +import math +import threading +import uuid +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final +from urllib.parse import urlsplit + +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue, TypeAdapter + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_VECTOR_SIZE: Final = 16 +_CHAT_MODEL: Final = "capped-chat" +_CLAUDE_MODEL: Final = "capped-claude" +_EMBEDDING_MODEL: Final = "capped-embedder" +_COLLECTION: Final = "cache-max-messages" + + +def _vector(text: str) -> tuple[float, ...]: + digest: Final = hashlib.sha256(text.encode()).digest() + return tuple((byte - 127.5) / 127.5 for byte in digest[:_VECTOR_SIZE]) + + +def _cosine(left: Sequence[float], right: Sequence[float]) -> float: + dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) + return dot / (math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))) + + +def _answer() -> str: + return f"answer-{uuid.uuid4().hex[:16]}" + + +@dataclass(slots=True) +class _Peer: + """Embeddings, chat, Responses, Anthropic Messages and a Qdrant collection on one owned socket""" + + lock: threading.Lock = field(default_factory=threading.Lock) + points: list[Mapping[str, JsonValue]] = field(default_factory=list) # mutable-ok: the Qdrant collection + embedded: list[str] = field(default_factory=list) # mutable-ok: every prompt the proxy embedded + answered: list[str] = field(default_factory=list) # mutable-ok: every completion the provider served + + def stored(self) -> int: + with self.lock: + return len(self.points) + + def respond(self, request: Request) -> Reply: + path: Final = urlsplit(request.target).path + body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} + collection: Final = f"/qdrant/collections/{_COLLECTION}" + if request.method == "GET" and path == f"{collection}/exists": + return self._json({"result": {"exists": False}, "status": "ok"}) + if request.method in {"GET", "PUT"} and path in {collection, f"{collection}/index"}: + return self._json({"result": True, "status": "ok"}) + if request.method == "PUT" and path == f"{collection}/points": + with self.lock: + self.points.extend(_JSON_OBJECT.validate_python(point) for point in _list(body["points"])) + return self._json({"result": {"status": "completed"}, "status": "ok"}) + if request.method == "POST" and path == f"{collection}/points/search": + return self._json({"result": self._search(body), "status": "ok"}) + if request.method == "GET" and path == "/v1/models": + return self._json({"object": "list", "data": []}) + if request.method == "POST" and path == "/v1/embeddings": + text: Final = str(body["input"]) + with self.lock: + self.embedded.append(text) + return self._json( + { + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": list(_vector(text))}], + "usage": {"prompt_tokens": 4, "total_tokens": 4}, + } + ) + if request.method == "POST" and path == "/v1/chat/completions": + return self._json(self._chat_reply(_answer())) + if request.method == "POST" and path == "/v1/responses": + return self._json(self._responses_reply(_answer())) + if request.method == "POST" and path == "/v1/messages": + return self._json(self._messages_reply(_answer())) + raise AssertionError(f"unexpected peer request {request.method} {request.target}") + + def _search(self, body: Mapping[str, JsonValue]) -> list[JsonValue]: + query: Final = [float(str(value)) for value in _list(body["vector"])] + key: Final = _JSON_OBJECT.validate_python( + _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_python(body["filter"])["must"])[0])["match"] + )["value"] + with self.lock: + scoped: Final = [ + point + for point in self.points + if _JSON_OBJECT.validate_python(point["payload"])["litellm_cache_key"] == key + ] + ranked: Final = sorted( + ( + { + "id": point["id"], + "score": _cosine(query, [float(str(value)) for value in _list(point["vector"])]), + "payload": point["payload"], + } + for point in scoped + ), + key=lambda hit: -float(str(hit["score"])), + ) + return list(ranked[:1]) + + def _chat_reply(self, answer: str) -> Mapping[str, JsonValue]: + with self.lock: + self.answered.append(answer) + return { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": 1789788253, + "model": "gpt-5.4-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, + } + + def _responses_reply(self, answer: str) -> Mapping[str, JsonValue]: + with self.lock: + self.answered.append(answer) + identity: Final = uuid.uuid4().hex + return { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1789788253, + "status": "completed", + "model": "gpt-5.4-mini", + "output": [ + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": answer, "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, + } + + def _messages_reply(self, answer: str) -> Mapping[str, JsonValue]: + with self.lock: + self.answered.append(answer) + return { + "id": f"msg_{uuid.uuid4().hex}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": answer}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 2}, + } + + @staticmethod + def _json(value: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(value).encode()) + + +def _list(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +@dataclass(frozen=True, slots=True) +class _Proxy: + gateway: Gateway + peer: _Peer + + def reply(self, route: str, conversation: list[JsonValue]) -> str: + """The answer text for one request, as the client sees it""" + model: Final = _CLAUDE_MODEL if route == "/v1/messages" else _CHAT_MODEL + body: Final[dict[str, JsonValue]] = ( + {"model": model, "input": conversation} + if route == "/v1/responses" + else {"model": model, "max_tokens": 16, "messages": conversation} + ) + response: Final = self.gateway.request("POST", route, body) + assert response.status_code == 200, response.text + payload: Final = _JSON_OBJECT.validate_json(response.content) + if route == "/v1/messages": + return str(_JSON_OBJECT.validate_python(_list(payload["content"])[0])["text"]) + if route == "/v1/responses": + message: Final = _JSON_OBJECT.validate_python(_list(payload["output"])[0]) + return str(_JSON_OBJECT.validate_python(_list(message["content"])[0])["text"]) + choice: Final = _JSON_OBJECT.validate_python(_list(payload["choices"])[0]) + return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) + + def miss(self, route: str, conversation: list[JsonValue]) -> str: + answered_before: Final = len(self.peer.answered) + answer: Final = self.reply(route, conversation) + assert len(self.peer.answered) == answered_before + 1, f"{answer} was served from the cache" + return answer + + def hit(self, route: str, conversation: list[JsonValue]) -> str: + """Repeats the request until the cache serves it, since the store after a miss is asynchronous""" + + def attempt() -> tuple[str, bool]: + answered_before: Final = len(self.peer.answered) + answer: Final = self.reply(route, conversation) + return answer, len(self.peer.answered) == answered_before + + return eventually(attempt, lambda outcome: outcome[1])[0] + + +def _proxy(tmp_path_factory: pytest.TempPathFactory, cache_params: Mapping[str, JsonValue]) -> Iterator[_Proxy]: + peer: Final = _Peer() + directory: Final = tmp_path_factory.mktemp("cache-max-messages") + with gateway_from_environment() as gateway, wire_server(peer.respond) as wire: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["model_list"] = [ + { + "model_name": _CHAT_MODEL, + "litellm_params": {"model": "openai/gpt-5.4-mini", "api_base": f"{wire.url}/v1", "api_key": "k"}, + }, + { + "model_name": _CLAUDE_MODEL, + "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_base": wire.url, "api_key": "k"}, + }, + { + "model_name": _EMBEDDING_MODEL, + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_base": f"{wire.url}/v1", + "api_key": "k", + }, + }, + ] + config["litellm_settings"]["cache_params"] = { + **cache_params, + **({"qdrant_api_base": f"{wire.url}/qdrant"} if cache_params["type"] == "qdrant-semantic" else {}), + } + path: Final = directory / "cache_max_messages.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, directory, {}, config=path) as candidate: + yield _Proxy(candidate, peer) + + +@pytest.fixture(scope="module") +def qdrant_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Proxy]: + """Qdrant semantic cache with the default max_messages of 4""" + yield from _proxy( + tmp_path_factory, + { + "type": "qdrant-semantic", + "qdrant_collection_name": _COLLECTION, + "qdrant_semantic_cache_embedding_model": _EMBEDDING_MODEL, + "qdrant_semantic_cache_vector_size": _VECTOR_SIZE, + "qdrant_quantization_config": "binary", + "similarity_threshold": 0.99, + }, + ) + + +@pytest.fixture(scope="module") +def exact_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Proxy]: + """Exact cache with max_messages lowered to 2 in cache_params""" + yield from _proxy(tmp_path_factory, {"type": "local", "max_messages": 2}) + + +def _claude_code_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: + """Turns 1 to 3 of a Claude Code session on /v1/messages: 1, 3 and 5 messages""" + first: Final[list[JsonValue]] = [{"role": "user", "content": task}] + second: Final[list[JsonValue]] = [ + *first, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, + ] + third: Final[list[JsonValue]] = [ + *second, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_2", "name": "Read", "input": {"file_path": "calc.py"}}], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_2", + "content": [{"type": "text", "text": "def add(a, b): return a - b"}], + } + ], + }, + ] + return first, second, third + + +def _agent_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: + """Turns 1 to 3 of an OpenAI tool loop on /v1/chat/completions: 2, 4 and 6 messages""" + + def call(call_id: str, path: str) -> list[JsonValue]: + return [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": call_id, + "type": "function", + "function": {"name": "write_file", "arguments": json.dumps({"path": path})}, + } + ], + }, + {"role": "tool", "tool_call_id": call_id, "content": f"wrote {path}"}, + ] + + first: Final[list[JsonValue]] = [ + {"role": "system", "content": "You are a coding agent"}, + {"role": "user", "content": task}, + ] + second: Final[list[JsonValue]] = [*first, *call("call_1", "a.yaml")] + third: Final[list[JsonValue]] = [*second, *call("call_2", "b.yaml")] + return first, second, third + + +def _responses_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: + """Turns 1 to 3 of an agent on /v1/responses: 1, 3 and 5 input items""" + + def call(call_id: str, path: str) -> list[JsonValue]: + return [ + { + "type": "function_call", + "call_id": call_id, + "name": "write_file", + "arguments": json.dumps({"path": path}), + }, + {"type": "function_call_output", "call_id": call_id, "output": f"wrote {path}"}, + ] + + first: Final[list[JsonValue]] = [{"role": "user", "content": task}] + second: Final[list[JsonValue]] = [*first, *call("call_1", "a.yaml")] + third: Final[list[JsonValue]] = [*second, *call("call_2", "b.yaml")] + return first, second, third + + +def _assert_turns_are_cached_up_to_four_messages( + proxy: _Proxy, route: str, turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]] +) -> None: + first, second, third = turns(f"update the config {uuid.uuid4().hex}") + stored_before: Final = proxy.peer.stored() + + short_answers: Final = (proxy.miss(route, first), proxy.miss(route, second)) + repeated: Final = (proxy.hit(route, first), proxy.hit(route, second)) + long_answers: Final = (proxy.miss(route, third), proxy.miss(route, third)) + proxy.hit(route, first) + + assert repeated == short_answers, f"a repeated short turn got another turn's answer: {short_answers} {repeated}" + assert long_answers[0] != long_answers[1], "a turn past max_messages was served from the cache" + assert all("b.yaml" not in text and "def add" not in text for text in proxy.peer.embedded), ( + "a turn past max_messages was embedded" + ) + assert proxy.peer.stored() == stored_before + 2, "a turn past max_messages was written to the cache" + + +@pytest.mark.parametrize( + ("route", "turns"), + [ + pytest.param("/v1/messages", _claude_code_turns, id="messages"), + pytest.param("/v1/chat/completions", _agent_turns, id="chat-completions"), + ], +) +def test_agent_turns_are_cached_up_to_four_messages_and_bypass_the_cache_past_it( + qdrant_proxy: _Proxy, + route: str, + turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]], +) -> None: + _assert_turns_are_cached_up_to_four_messages(qdrant_proxy, route, turns) + + +def test_tool_result_text_tells_a_tool_turn_apart_from_the_turn_before_it(qdrant_proxy: _Proxy) -> None: + task: Final = f"list the files {uuid.uuid4().hex}" + first, second, _ = _claude_code_turns(task) + + qdrant_proxy.miss("/v1/messages", first) + qdrant_proxy.miss("/v1/messages", second) + + assert task in qdrant_proxy.peer.embedded[-1], qdrant_proxy.peer.embedded[-1] + assert "calc.py test_calc.py" in qdrant_proxy.peer.embedded[-1], qdrant_proxy.peer.embedded[-1] + + +@pytest.mark.parametrize( + ("route", "turns"), + [ + pytest.param("/v1/messages", _claude_code_turns, id="messages"), + pytest.param("/v1/chat/completions", _agent_turns, id="chat-completions"), + pytest.param("/v1/responses", _responses_turns, id="responses"), + ], +) +def test_exact_cache_honours_max_messages_from_cache_params( + exact_proxy: _Proxy, + route: str, + turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]], +) -> None: + first, second, _ = turns(f"what is {uuid.uuid4().hex}") + + answer: Final = exact_proxy.miss(route, first) + repeated: Final = exact_proxy.hit(route, first) + long_answers: Final = (exact_proxy.miss(route, second), exact_proxy.miss(route, second)) + + assert repeated == answer + assert long_answers[0] != long_answers[1], "a turn past max_messages was served from a cache capped at 2" diff --git a/tests/integration/caching/test_semantic_cache_tool_turns.py b/tests/integration/caching/test_semantic_cache_tool_turns.py deleted file mode 100644 index ac80b806c402..000000000000 --- a/tests/integration/caching/test_semantic_cache_tool_turns.py +++ /dev/null @@ -1,544 +0,0 @@ -import asyncio -import hashlib -import json -import math -import threading -import uuid -from collections.abc import Callable, Iterator, Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass, field -from pathlib import Path -from typing import Final -from urllib.parse import urlsplit - -import anthropic -import httpx -import openai -import pytest -import yaml -from integration._support.client import Gateway, eventually, gateway_from_environment -from integration._support.process import owned_proxy -from integration._support.wire import Reply, Request, wire_server -from pydantic import JsonValue, TypeAdapter - -_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) -_VECTOR_SIZE: Final = 16 -_CHAT_MODEL: Final = "semantic-chat" -_CLAUDE_MODEL: Final = "semantic-claude" -_EMBEDDING_MODEL: Final = "semantic-embedder" -_COLLECTION: Final = "semantic-tool-turns" - - -def _vector(text: str) -> tuple[float, ...]: - digest: Final = hashlib.sha256(text.encode()).digest() - return tuple((byte - 127.5) / 127.5 for byte in digest[:_VECTOR_SIZE]) - - -def _cosine(left: Sequence[float], right: Sequence[float]) -> float: - dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) - return dot / (math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))) - - -def _answer(body: Mapping[str, JsonValue]) -> str: - return "answer-" + hashlib.sha256(json.dumps(body["messages"], sort_keys=True).encode()).hexdigest()[:16] - - -@dataclass(slots=True) -class _Peer: - """Embeddings, chat, Anthropic Messages and a Qdrant collection on one owned socket""" - - lock: threading.Lock = field(default_factory=threading.Lock) - points: list[Mapping[str, JsonValue]] = field(default_factory=list) # mutable-ok: the Qdrant collection - embedded: list[str] = field(default_factory=list) # mutable-ok: every prompt the proxy embedded - answered: list[str] = field(default_factory=list) # mutable-ok: every completion the provider served - qdrant_down: threading.Event = field(default_factory=threading.Event) - refused_writes: list[str] = field(default_factory=list) # mutable-ok: upserts refused while Qdrant is down - - def stored(self) -> int: - with self.lock: - return len(self.points) - - def respond(self, request: Request) -> Reply: - path: Final = urlsplit(request.target).path - body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} - collection: Final = f"/qdrant/collections/{_COLLECTION}" - if path.startswith("/qdrant/") and self.qdrant_down.is_set(): - if path == f"{collection}/points": - with self.lock: - self.refused_writes.append(path) - return Reply(status=503, body=b'{"status":{"error":"qdrant unavailable"}}') - if request.method == "GET" and path == f"{collection}/exists": - return self._json({"result": {"exists": False}, "status": "ok"}) - if request.method in {"GET", "PUT"} and path in {collection, f"{collection}/index"}: - return self._json({"result": True, "status": "ok"}) - if request.method == "PUT" and path == f"{collection}/points": - with self.lock: - self.points.extend(_JSON_OBJECT.validate_python(point) for point in body["points"]) - return self._json({"result": {"status": "completed"}, "status": "ok"}) - if request.method == "POST" and path == f"{collection}/points/search": - return self._json({"result": self._search(body), "status": "ok"}) - if request.method == "GET" and path == "/v1/models": - return self._json({"object": "list", "data": []}) - if request.method == "POST" and path == "/v1/embeddings": - text: Final = str(body["input"]) - with self.lock: - self.embedded.append(text) - return self._json( - { - "object": "list", - "model": "text-embedding-3-small", - "data": [{"object": "embedding", "index": 0, "embedding": list(_vector(text))}], - "usage": {"prompt_tokens": 4, "total_tokens": 4}, - } - ) - if request.method == "POST" and path == "/v1/chat/completions": - return self._json(self._chat_reply(_answer(body))) - if request.method == "POST" and path == "/v1/messages": - return self._json(self._messages_reply(_answer(body))) - raise AssertionError(f"unexpected peer request {request.method} {request.target}") - - def _search(self, body: Mapping[str, JsonValue]) -> list[JsonValue]: - query: Final = [float(str(value)) for value in _list(body["vector"])] - key: Final = _JSON_OBJECT.validate_python( - _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_python(body["filter"])["must"])[0])["match"] - )["value"] - with self.lock: - scoped: Final = [ - point - for point in self.points - if _JSON_OBJECT.validate_python(point["payload"])["litellm_cache_key"] == key - ] - ranked: Final = sorted( - ( - { - "id": point["id"], - "score": _cosine(query, [float(str(value)) for value in _list(point["vector"])]), - "payload": point["payload"], - } - for point in scoped - ), - key=lambda hit: -float(str(hit["score"])), - ) - return list(ranked[:1]) - - def _chat_reply(self, answer: str) -> Mapping[str, JsonValue]: - with self.lock: - self.answered.append(answer) - return { - "id": f"chatcmpl-{uuid.uuid4().hex}", - "object": "chat.completion", - "created": 1789788253, - "model": "gpt-5.4-mini", - "choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, - } - - def _messages_reply(self, answer: str) -> Mapping[str, JsonValue]: - with self.lock: - self.answered.append(answer) - return { - "id": f"msg_{uuid.uuid4().hex}", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-5-5", - "content": [{"type": "text", "text": answer}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": 10, "output_tokens": 2}, - } - - @staticmethod - def _json(value: Mapping[str, JsonValue]) -> Reply: - return Reply(body=json.dumps(value).encode()) - - -def _list(value: JsonValue) -> list[JsonValue]: - assert isinstance(value, list), value - return value - - -@dataclass(frozen=True, slots=True) -class _SemanticProxy: - gateway: Gateway - peer: _Peer - - -@pytest.fixture(scope="module") -def semantic_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_SemanticProxy]: - peer: Final = _Peer() - directory: Final = tmp_path_factory.mktemp("semantic-tool-turns") - with gateway_from_environment() as gateway, wire_server(peer.respond) as wire: - config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) - config["model_list"] = [ - { - "model_name": _CHAT_MODEL, - "litellm_params": {"model": "openai/gpt-5.4-mini", "api_base": f"{wire.url}/v1", "api_key": "k"}, - }, - { - "model_name": _CLAUDE_MODEL, - "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_base": wire.url, "api_key": "k"}, - }, - { - "model_name": _EMBEDDING_MODEL, - "litellm_params": { - "model": "openai/text-embedding-3-small", - "api_base": f"{wire.url}/v1", - "api_key": "k", - }, - }, - ] - config["litellm_settings"]["cache_params"] = { - "type": "qdrant-semantic", - "qdrant_api_base": f"{wire.url}/qdrant", - "qdrant_collection_name": _COLLECTION, - "qdrant_semantic_cache_embedding_model": _EMBEDDING_MODEL, - "qdrant_semantic_cache_vector_size": _VECTOR_SIZE, - "qdrant_quantization_config": "binary", - "similarity_threshold": 0.99, - } - path: Final = directory / "semantic_tool_turns.yaml" - path.write_text(yaml.safe_dump(config)) - with owned_proxy(gateway, directory, {}, config=path) as candidate: - yield _SemanticProxy(candidate, peer) - - -def _send_turns(proxy: _SemanticProxy, route: str, model: str, turns: Sequence[list[JsonValue]]) -> list[str]: - def reply_text(turn: list[JsonValue]) -> str: - stored_before: Final = proxy.peer.stored() - answered_before: Final = len(proxy.peer.answered) - body: Final[dict[str, JsonValue]] = {"model": model, "max_tokens": 16, "messages": turn} - response: Final = proxy.gateway.request("POST", route, body) - assert response.status_code == 200, response.text - payload: Final = _JSON_OBJECT.validate_json(response.content) - if len(proxy.peer.answered) > answered_before: - eventually(proxy.peer.stored, lambda count: count > stored_before) - if route == "/v1/messages": - return str(_JSON_OBJECT.validate_python(_list(payload["content"])[0])["text"]) - choice: Final = _JSON_OBJECT.validate_python(_list(payload["choices"])[0]) - return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) - - return [reply_text(turn) for turn in turns] - - -def test_claude_code_tool_turns_on_messages_are_not_served_the_first_turn_answer( - semantic_proxy: _SemanticProxy, -) -> None: - task: Final[JsonValue] = {"role": "user", "content": f"fix the failing test {uuid.uuid4().hex}"} - list_files: Final[list[JsonValue]] = [ - task, - { - "role": "assistant", - "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], - }, - { - "role": "user", - "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], - }, - ] - read_file: Final[list[JsonValue]] = [ - *list_files, - { - "role": "assistant", - "content": [{"type": "tool_use", "id": "toolu_2", "name": "Read", "input": {"file_path": "calc.py"}}], - }, - { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_2", - "content": [{"type": "text", "text": "def add(a, b): return a - b"}], - } - ], - }, - ] - - answers: Final = _send_turns( - semantic_proxy, "/v1/messages", _CLAUDE_MODEL, ([task], list_files, read_file, list_files) - ) - - assert len(set(answers[:3])) == 3, f"a later tool turn replayed an earlier cached answer: {answers}" - assert answers[:3] == semantic_proxy.peer.answered[-3:], semantic_proxy.peer.answered - assert answers[3] == answers[1], f"a repeated tool turn missed the cache: {answers}" - - -def test_openai_agent_loop_tool_calls_on_chat_completions_are_not_served_a_cached_answer( - semantic_proxy: _SemanticProxy, -) -> None: - task: Final[JsonValue] = {"role": "user", "content": f"update both config files {uuid.uuid4().hex}"} - - def wrote(path: str) -> list[JsonValue]: - return [ - task, - { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": {"name": "write_file", "arguments": json.dumps({"path": path})}, - } - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": "ok"}, - ] - - answers: Final = _send_turns( - semantic_proxy, "/v1/chat/completions", _CHAT_MODEL, (wrote("a.yaml"), wrote("b.yaml"), wrote("a.yaml")) - ) - - assert len(set(answers[:2])) == 2, f"a different tool call replayed an earlier cached answer: {answers}" - assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered - assert answers[2] == answers[0], f"a repeated tool call missed the cache: {answers}" - - -def _tool_turn(task: str, calls: Sequence[tuple[str, str]], results: Sequence[tuple[str, str]]) -> list[JsonValue]: - return [ - {"role": "user", "content": task}, - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": call_id, "type": "function", "function": {"name": "read_file", "arguments": arguments}} - for call_id, arguments in calls - ], - }, - *({"role": "tool", "tool_call_id": call_id, "content": output} for call_id, output in results), - ] - - -def _claude_turn(task: str, calls: Sequence[tuple[str, str]], results: Sequence[tuple[str, str]]) -> list[JsonValue]: - return [ - {"role": "user", "content": task}, - { - "role": "assistant", - "content": [ - {"type": "tool_use", "id": call_id, "name": "Read", "input": {"file_path": file_path}} - for call_id, file_path in calls - ], - }, - { - "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": call_id, "content": output} for call_id, output in results - ], - }, - ] - - -_TWO_READS: Final = (("c1", '{"path":"a.py"}'), ("c2", '{"path":"b.py"}')) -_TWO_CLAUDE_READS: Final = (("toolu_1", "a.py"), ("toolu_2", "b.py")) - - -@pytest.mark.parametrize( - ("route", "model", "build", "calls", "first", "second"), - [ - pytest.param( - "/v1/chat/completions", - _CHAT_MODEL, - _tool_turn, - _TWO_READS, - (("c1", "x = 1"), ("c2", "y = 2")), - (("c2", "x = 1"), ("c1", "y = 2")), - id="chat-parallel-results-answer-the-other-call", - ), - pytest.param( - "/v1/messages", - _CLAUDE_MODEL, - _claude_turn, - _TWO_CLAUDE_READS, - (("toolu_1", "x = 1"), ("toolu_2", "y = 2")), - (("toolu_2", "x = 1"), ("toolu_1", "y = 2")), - id="messages-parallel-results-answer-the-other-call", - ), - pytest.param( - "/v1/chat/completions", - _CHAT_MODEL, - _tool_turn, - _TWO_READS[:1], - (("c1", "x = 1"),), - (("c9", "x = 1"),), - id="chat-result-for-an-unknown-call", - ), - pytest.param( - "/v1/chat/completions", - _CHAT_MODEL, - _tool_turn, - _TWO_READS, - (("c1", '"},{"result_of_call":2,"output":"y = 2'),), - (("c1", ""), ("c2", "y = 2")), - id="chat-output-shaped-like-a-second-result", - ), - pytest.param( - "/v1/chat/completions", - _CHAT_MODEL, - _tool_turn, - _TWO_READS[:1], - (("c1", ""),), - (("c1", "x" * 5000),), - id="chat-empty-and-5kb-output", - ), - ], -) -def test_tool_turns_that_differ_only_in_their_results_do_not_share_a_cached_answer( - semantic_proxy: _SemanticProxy, - route: str, - model: str, - build: Callable[[str, Sequence[tuple[str, str]], Sequence[tuple[str, str]]], list[JsonValue]], - calls: Sequence[tuple[str, str]], - first: Sequence[tuple[str, str]], - second: Sequence[tuple[str, str]], -) -> None: - task: Final = f"read both files {uuid.uuid4().hex}" - - answers: Final = _send_turns( - semantic_proxy, route, model, (build(task, calls, first), build(task, calls, second), build(task, calls, first)) - ) - - assert answers[0] != answers[1], f"a different tool result replayed an earlier cached answer: {answers}" - assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered - assert answers[2] == answers[0], f"a repeated tool turn missed the cache: {answers}" - - -def _openai_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: - sdk: Final = openai.OpenAI( - base_url=str(proxy.gateway.client.base_url) + "/v1", - api_key=proxy.gateway.key, - http_client=httpx.Client(trust_env=False, timeout=15), - ) - completion: Final = sdk.chat.completions.create(model=_CHAT_MODEL, messages=turn) # pyright: ignore[reportArgumentType, reportCallIssue] # JSON turns are validated by the proxy, not the SDK types - return str(completion.choices[0].message.content) - - -def _async_openai_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: - async def create() -> str: - async with httpx.AsyncClient(trust_env=False, timeout=15) as http_client: - sdk: Final = openai.AsyncOpenAI( - base_url=str(proxy.gateway.client.base_url) + "/v1", api_key=proxy.gateway.key, http_client=http_client - ) - completion: Final = await sdk.chat.completions.create(model=_CHAT_MODEL, messages=turn) # pyright: ignore[reportArgumentType, reportCallIssue] # JSON turns are validated by the proxy, not the SDK types - return str(completion.choices[0].message.content) - - return asyncio.run(create()) - - -def _anthropic_text(message: anthropic.types.Message) -> str: - block: Final = message.content[0] - assert isinstance(block, anthropic.types.TextBlock), message - return block.text - - -def _anthropic_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: - sdk: Final = anthropic.Anthropic( - base_url=str(proxy.gateway.client.base_url), - api_key=proxy.gateway.key, - http_client=httpx.Client(trust_env=False, timeout=15), - ) - return _anthropic_text(sdk.messages.create(model=_CLAUDE_MODEL, max_tokens=16, messages=turn)) # pyright: ignore[reportArgumentType] # JSON turns are validated by the proxy, not the SDK types - - -def _async_anthropic_sdk(proxy: _SemanticProxy, turn: list[JsonValue]) -> str: - async def create() -> str: - async with httpx.AsyncClient(trust_env=False, timeout=15) as http_client: - sdk: Final = anthropic.AsyncAnthropic( - base_url=str(proxy.gateway.client.base_url), api_key=proxy.gateway.key, http_client=http_client - ) - message: Final = await sdk.messages.create(model=_CLAUDE_MODEL, max_tokens=16, messages=turn) # pyright: ignore[reportArgumentType] # JSON turns are validated by the proxy, not the SDK types - return _anthropic_text(message) - - return asyncio.run(create()) - - -@pytest.mark.parametrize( - ("send", "build"), - [ - pytest.param(_openai_sdk, _tool_turn, id="openai-sdk"), - pytest.param(_async_openai_sdk, _tool_turn, id="async-openai-sdk"), - pytest.param(_anthropic_sdk, _claude_turn, id="anthropic-sdk"), - pytest.param(_async_anthropic_sdk, _claude_turn, id="async-anthropic-sdk"), - ], -) -def test_sdk_clients_get_a_fresh_answer_for_each_tool_turn_and_a_hit_for_a_repeat( - semantic_proxy: _SemanticProxy, - send: Callable[[_SemanticProxy, list[JsonValue]], str], - build: Callable[[str, Sequence[tuple[str, str]], Sequence[tuple[str, str]]], list[JsonValue]], -) -> None: - task: Final = f"read the file {uuid.uuid4().hex}" - calls: Final = _TWO_READS[:1] if build is _tool_turn else _TWO_CLAUDE_READS[:1] - call_id: Final = calls[0][0] - alpha: Final = build(task, calls, ((call_id, "x = 1"),)) - beta: Final = build(task, calls, ((call_id, "PermissionError"),)) - - def answer(turn: list[JsonValue]) -> str: - stored_before: Final = semantic_proxy.peer.stored() - answered_before: Final = len(semantic_proxy.peer.answered) - text: Final = send(semantic_proxy, turn) - if len(semantic_proxy.peer.answered) > answered_before: - eventually(semantic_proxy.peer.stored, lambda count: count > stored_before) - return text - - answers: Final = [answer(turn) for turn in (alpha, beta, alpha)] - - assert answers[0] != answers[1], f"a different tool result replayed an earlier cached answer: {answers}" - assert answers[:2] == semantic_proxy.peer.answered[-2:], semantic_proxy.peer.answered - assert answers[2] == answers[0], f"a repeated tool turn missed the cache: {answers}" - - -def test_concurrent_agent_loops_each_get_their_own_answer_and_their_own_hit(semantic_proxy: _SemanticProxy) -> None: - task: Final = f"read the file {uuid.uuid4().hex}" - turns: Final = tuple(_tool_turn(task, _TWO_READS[:1], (("c1", f"line {index}"),)) for index in range(8)) - - def send(turn: list[JsonValue]) -> str: - response: Final = semantic_proxy.gateway.request( - "POST", "/v1/chat/completions", {"model": _CHAT_MODEL, "messages": turn} - ) - assert response.status_code == 200, response.text - choice: Final = _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_json(response.content)["choices"])[0]) - return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) - - stored_before: Final = semantic_proxy.peer.stored() - with ThreadPoolExecutor(max_workers=len(turns)) as pool: - first: Final = tuple(pool.map(send, turns)) - eventually(semantic_proxy.peer.stored, lambda count: count >= stored_before + len(turns)) - answered_before: Final = len(semantic_proxy.peer.answered) - with ThreadPoolExecutor(max_workers=len(turns)) as pool: - second: Final = tuple(pool.map(send, turns)) - - assert len(set(first)) == len(turns), f"concurrent tool turns shared a cached answer: {first}" - assert second == first, f"a repeated tool turn got another turn's answer: {first} {second}" - assert len(semantic_proxy.peer.answered) == answered_before, "a repeated tool turn missed the cache" - - -def test_tool_turns_are_answered_while_qdrant_is_down_and_cached_once_it_recovers( - semantic_proxy: _SemanticProxy, -) -> None: - task: Final = f"read the file {uuid.uuid4().hex}" - turn: Final = _tool_turn(task, _TWO_READS[:1], (("c1", "x = 1"),)) - body: Final[dict[str, JsonValue]] = {"model": _CHAT_MODEL, "messages": turn} - - def send() -> httpx.Response: - return semantic_proxy.gateway.request("POST", "/v1/chat/completions", body) - - refused_before: Final = len(semantic_proxy.peer.refused_writes) - answered_before: Final = len(semantic_proxy.peer.answered) - semantic_proxy.peer.qdrant_down.set() - try: - outage: Final = (send(), send()) - eventually(lambda: len(semantic_proxy.peer.refused_writes), lambda count: count >= refused_before + 2) - finally: - semantic_proxy.peer.qdrant_down.clear() - answered_during_outage: Final = len(semantic_proxy.peer.answered) - answered_before - stored_before: Final = semantic_proxy.peer.stored() - recovered: Final = send() - eventually(semantic_proxy.peer.stored, lambda count: count > stored_before) - answered_after_recovery: Final = len(semantic_proxy.peer.answered) - repeat: Final = send() - - assert [response.status_code for response in outage] == [200, 200], [response.text for response in outage] - assert answered_during_outage == 2, "an outage request was not sent to the provider" - assert recovered.status_code == 200, recovered.text - assert answered_after_recovery - answered_before == 3, "the first request after recovery hit a stale entry" - assert repeat.status_code == 200, repeat.text - assert repeat.json()["choices"] == recovered.json()["choices"] - assert len(semantic_proxy.peer.answered) == answered_after_recovery, "the repeat after recovery missed the cache" diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index 0a7ac3ecad16..0adb6b9a6f42 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -1,6 +1,7 @@ import asyncio import logging import re +import uuid from typing import Final from unittest.mock import MagicMock @@ -9,7 +10,7 @@ import litellm import litellm.caching.redis_cache as redis_cache_module from litellm._internal_context import current_service_target -from litellm.caching.caching import Cache, response_cache_phase +from litellm.caching.caching import Cache, CacheMode, response_cache_phase from litellm.caching.caching_handler import _PENDING_CACHE_WRITES from litellm.caching.in_memory_cache import InMemoryCache from litellm.caching.redis_cache import RedisCache, _RedisTimeoutLogThrottle @@ -473,3 +474,65 @@ async def test_a_lookup_already_inside_the_phase_does_not_open_a_second_one(v2_s await cache.async_get_cache(dynamic_cache_object=backend, **_REQUEST) assert backend.seen == [("llm_response", "cache.get llm_response")] assert [s.name for s in v2_span_exporter.get_finished_spans()] == ["cache.get llm_response"] + + +_TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"} + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + [ + pytest.param({"messages": [_TOOL_TURN_ITEM] * 4}, True, id="four-messages-are-cached"), + pytest.param({"messages": [_TOOL_TURN_ITEM] * 5}, False, id="five-messages-skip-the-cache"), + pytest.param({"input": [_TOOL_TURN_ITEM] * 4}, True, id="four-responses-items-are-cached"), + pytest.param({"input": [_TOOL_TURN_ITEM] * 5}, False, id="five-responses-items-skip-the-cache"), + pytest.param({"input": "one prompt"}, True, id="string-input-is-one-message"), + pytest.param({"input": ["a", "b", "c", "d", "e"]}, True, id="embedding-strings-are-not-messages"), + ], +) +def test_should_use_cache_stops_past_the_default_max_messages(kwargs: dict[str, object], expected: bool) -> None: + assert Cache(type=LiteLLMCacheType.LOCAL).should_use_cache(**kwargs) is expected + + +def test_responses_sdk_items_count_toward_max_messages() -> None: + from openai.types.responses import ResponseFunctionToolCall + + call: Final = ResponseFunctionToolCall(type="function_call", call_id="c1", name="ls", arguments="{}") + + assert Cache(type=LiteLLMCacheType.LOCAL).should_use_cache(input=[_TOOL_TURN_ITEM, call, call, call, call]) is False + + +def test_max_messages_is_configurable_and_none_disables_it() -> None: + three: Final = [_TOOL_TURN_ITEM] * 3 + + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=2).should_use_cache(messages=three) is False + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=3).should_use_cache(messages=three) is True + assert Cache(type=LiteLLMCacheType.LOCAL, max_messages=None).should_use_cache(messages=three * 50) is True + + +def test_max_messages_beats_an_explicit_use_cache_opt_in() -> None: + cache: Final = Cache(type=LiteLLMCacheType.LOCAL, mode=CacheMode.default_off) + + assert cache.should_use_cache(messages=[_TOOL_TURN_ITEM] * 4, cache={"use-cache": True}) is True + assert cache.should_use_cache(messages=[_TOOL_TURN_ITEM] * 5, cache={"use-cache": True}) is False + + +def test_completion_past_max_messages_is_neither_served_from_nor_written_to_the_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "cache", Cache(type=LiteLLMCacheType.LOCAL)) + tag: Final = uuid.uuid4().hex + four: Final = [{"role": "user", "content": f"{tag} turn {index}"} for index in range(4)] + five: Final = [*four, {"role": "user", "content": f"{tag} turn 4"}] + + def answer(messages: list[dict[str, str]], mock_response: str) -> str: + response: Final = litellm.completion(model="gpt-4o-mini", messages=messages, mock_response=mock_response) + assert isinstance(response, litellm.ModelResponse), response + choice: Final = response.choices[0] + assert isinstance(choice, litellm.Choices), choice + return str(choice.message.content) + + assert answer(four, "four first") == "four first" + assert answer(four, "four second") == "four first", "a 4-message repeat missed the cache" + assert answer(five, "five first") == "five first" + assert answer(five, "five second") == "five second", "a 5-message repeat was served from the cache" diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 07ec324699b7..61e5a727411d 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -567,134 +567,70 @@ def test_redis_semantic_cache_prompt_extraction_prefers_messages(): assert prompt == "message prompt" -def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct(): +def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - def turn(command: str) -> list[dict[str, object]]: - return [ - {"role": "user", "content": "fix the failing test"}, - { - "role": "assistant", - "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": command}}], - }, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]}, - ] - - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' - ) - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' - ) - + class ModelDumpInput: + def model_dump(self): + return {"content": [{"text": "model dump prompt"}]} -def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache + class DictInput: + def dict(self): + return {"content": [{"output_text": "dict prompt"}]} prompt = RedisSemanticCache._get_prompt_from_kwargs( input=[ - {"role": "user", "content": "update the config"}, - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path":"a.yaml"}'}, - {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ModelDumpInput(), + DictInput(), + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, ] ) - assert prompt == ( - 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' - ) + assert prompt == "model dump prompt\ndict prompt\ninline prompt" -def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_outputs(): +def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - def prompt_for(output_text: str) -> str | None: - return RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "write hello"}]}, - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path":"a.txt"}'}, - { - "type": "function_call_output", - "call_id": "c1", - "output": [{"type": "input_text", "text": output_text}], - }, - ] - ) - - expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' + assert RedisSemanticCache._get_prompt_from_kwargs() is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None + assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None assert ( - prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' + RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_image", "image_url": "https://example.com"}]) + is None ) - assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") -def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): +def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - { - "type": "function_call_output", - "call_id": "c1", - "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], - }, - ] - ) + prompt = RedisSemanticCache._get_prompt_from_kwargs(input={"text": " ", "input_text": "fallback prompt"}) - assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' + assert prompt == "fallback prompt" -def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): +def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - def prompt_for(first_output_call_id: str, second_output_call_id: str) -> str | None: - return RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, - {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, - {"type": "function_call_output", "call_id": first_output_call_id, "output": "empty"}, - {"type": "function_call_output", "call_id": second_output_call_id, "output": "secret"}, - ] - ) + class ResponseInput: + text = " " + input_text = "fallback prompt" - assert prompt_for("c2", "c1") == ( - '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' - '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' - ) - assert prompt_for("c1", "c2") != prompt_for("c2", "c1") + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + assert prompt == "fallback prompt" -def test_redis_semantic_cache_prompt_extraction_dumps_sdk_response_items_appended_to_input(): - from openai.types.responses import ResponseFunctionToolCall +def test_redis_semantic_cache_prompt_extraction_handles_object_content(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"role": "user", "content": "write hello"}, - ResponseFunctionToolCall( - type="function_call", call_id="c1", name="write_file", arguments='{"path":"a.txt"}' - ), - {"type": "function_call_output", "call_id": "c1", "output": "ok"}, - ] - ) - - assert ( - prompt - == 'write hello\n{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n{"result_of_call":1,"output":"ok"}' - ) + class ResponseInput: + content = [{"text": "object content prompt"}] + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) -def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - assert RedisSemanticCache._get_prompt_from_kwargs() is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None - assert ( - RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_image", "image_url": "https://example.com"}]) - is None - ) + assert prompt == "object content prompt" def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): @@ -1454,3 +1390,23 @@ async def test_redis_async_embedding_truncates_off_the_event_loop(monkeypatch): assert embedding == [0.1, 0.2] assert _token_count("sem-embed", router.aembedding.call_args.kwargs["input"]) == 5 assert_loop_stayed_free(took, lags) + + +def test_redis_semantic_cache_prompt_extraction_keeps_tool_result_text(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + messages=[ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, + ] + ) + + assert prompt == "list the filescalc.py test_calc.py" diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 61d7cdbd96c0..1d19264fb9e2 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -30,7 +30,6 @@ system_messages_first, update_messages_with_model_file_ids, ) -from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' _ARTIFACT_DATA_ID_PATTERN: Final = r"^(?!\.\.?(?:\/|$))[A-Za-z0-9_\-.~:@+]{1,200}$" @@ -2176,28 +2175,23 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s assert merged == [{"role": "system"}, {"role": "user", "content": "Hi"}] -_TASK: Final = {"role": "user", "content": "fix the failing test"} +_CLAUDE_CODE_TOOL_TURN: Final = [ + {"role": "user", "content": "list the files"}, + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], + }, +] @pytest.mark.parametrize( ("messages", "expected"), [ - pytest.param( - [ - _TASK, - { - "role": "assistant", - "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], - }, - ], - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', - id="anthropic-tool-use-name-and-input-without-id", - ), - pytest.param( - [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - 'fix the failing test{"result_of_call":null,"output":"calc.py"}', - id="anthropic-string-tool-result", - ), + pytest.param(_CLAUDE_CODE_TOOL_TURN, "list the filescalc.py test_calc.py", id="tool-result-string"), pytest.param( [ { @@ -2205,166 +2199,67 @@ def test_keeps_the_first_message_when_no_system_message_in_the_run_has_content(s "content": [ { "type": "tool_result", - "tool_use_id": "t1", - "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}], - }, - {"type": "text", "text": "next"}, + "tool_use_id": "toolu_1", + "content": [ + {"type": "text", "text": "x = 1"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": ""}}, + ], + } ], } ], - '{"result_of_call":null,"output":"ab"}next', - id="anthropic-nested-text-tool-result-then-text", + "x = 1", + id="tool-result-blocks", ), pytest.param( - [ - _TASK, - { - "role": "assistant", - "content": "writing", - "tool_calls": [ - {"id": "c1", "type": "function", "function": {"name": "write", "arguments": '{"path": "a"}'}}, - {"id": "c2", "type": "function", "function": {"name": "write", "arguments": '{"path": "b"}'}}, - ], - }, - {"role": "tool", "tool_call_id": "c1", "content": "ok"}, - ], - 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' - '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', - id="openai-tool-calls-in-order-before-tool-result", - ), - pytest.param( - [ - { - "role": "assistant", - "content": [ - {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, - {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, - ], - }, - { - "role": "user", - "content": [ - {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, - {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, - ], - }, - ], - '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' - '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', - id="anthropic-parallel-tool-results-tagged-with-their-call", - ), - pytest.param( - [ - { - "role": "assistant", - "content": None, - "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], - }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, - ], - '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' - '{"result_of_call":2,"output":"C"}', - id="reused-call-ids-keep-first-position", - ), - pytest.param( - [ - { - "role": "tool", - "tool_call_id": "c1", - "content": "small", - "search_results": [{"source": "s", "title": "t"}], - } - ], - '{"result_of_call":null,"output":"small"}st', - id="tool-result-search-results-follow-the-encoded-result", - ), - pytest.param( - [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], - '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', - id="tool-output-cannot-forge-an-encoded-result", - ), - pytest.param( - [ - Message( - content=None, - tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], - ) - ], - '{"name":"read","arguments":"{}"}', - id="openai-response-message-object", - ), - pytest.param( - [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], - '{"name":null,"arguments":null}', - id="malformed-tool-call-entries", + [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1"}]}], + "", + id="tool-result-without-content", ), ], ) -def test_get_semantic_cache_prompt_from_messages_keeps_tool_exchange(messages: list[object], expected: str) -> None: +def test_get_semantic_cache_prompt_from_messages_keeps_tool_result_text( + messages: list[dict[str, object]], expected: str +) -> None: assert get_semantic_cache_prompt_from_messages(messages) == expected +def test_get_semantic_cache_prompt_from_messages_differs_from_the_turn_before_it() -> None: + assert get_str_from_messages(_CLAUDE_CODE_TOOL_TURN) == get_str_from_messages(_CLAUDE_CODE_TOOL_TURN[:1]) + assert get_semantic_cache_prompt_from_messages(_CLAUDE_CODE_TOOL_TURN) != get_semantic_cache_prompt_from_messages( + _CLAUDE_CODE_TOOL_TURN[:1] + ) + + @pytest.mark.parametrize( "messages", [ - pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), + pytest.param([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}], id="strings"), pytest.param( [ { "role": "user", "content": [ - {"type": "text", "text": "what is "}, + {"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, - {"type": "text", "text": "this"}, + {"type": "text", "text": "this?"}, ], } ], - id="text-and-image-parts", + id="text-parts", + ), + pytest.param( + [ + {"role": "assistant"}, + {"role": "assistant", "content": None}, + {"role": "user", "content": ""}, + {"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": []}]}, + ], + id="empty-content-and-search-results", ), - pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_without_tools( - messages: list[object], +def test_get_semantic_cache_prompt_from_messages_matches_get_str_from_messages_without_tool_results( + messages: list[dict[str, object]], ) -> None: - assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures - - -def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: - return [ - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": f"{call_id_prefix}1", "type": "function", "function": {"name": "read", "arguments": '"a"'}}, - {"id": f"{call_id_prefix}2", "type": "function", "function": {"name": "read", "arguments": '"b"'}}, - ], - }, - {"role": "tool", "tool_call_id": f"{call_id_prefix}1", "content": result_for_a}, - {"role": "tool", "tool_call_id": f"{call_id_prefix}2", "content": result_for_b}, - ] - - -def _results_in_swapped_order(result_for_a: str, result_for_b: str) -> list[object]: - call, answer_a, answer_b = _parallel_reads(result_for_a, result_for_b) - return [call, answer_b, answer_a] - - -def test_get_semantic_cache_prompt_from_messages_tells_apart_parallel_results_answering_different_calls() -> None: - assert get_semantic_cache_prompt_from_messages( - _parallel_reads("empty", "secret") - ) != get_semantic_cache_prompt_from_messages(_results_in_swapped_order("secret", "empty")) - - -def test_get_semantic_cache_prompt_from_messages_ignores_call_ids_that_differ_between_sessions() -> None: - assert get_semantic_cache_prompt_from_messages( - _parallel_reads("A", "B", call_id_prefix="toolu_") - ) == get_semantic_cache_prompt_from_messages(_parallel_reads("A", "B")) + assert get_semantic_cache_prompt_from_messages(messages) == get_str_from_messages(messages) From 050e9c0418ea9bad7a34022de6d83e91f5e9dda0 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 23:31:08 +0000 Subject: [PATCH 22/29] test(caching): drop formatting-only churn from the redis semantic cache tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../unit/caching/test_redis_semantic_cache.py | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 61e5a727411d..9077cded532f 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -203,7 +203,10 @@ def test_redis_semantic_cache_uses_isolated_index_for_old_schema(monkeypatch): assert redis_semantic_cache.llmcache is fallback_cache_mock assert semantic_cache_mock.call_args_list[0].kwargs["name"] == "existing_index" - assert semantic_cache_mock.call_args_list[1].kwargs["name"] == "existing_index_isolated" + assert ( + semantic_cache_mock.call_args_list[1].kwargs["name"] + == "existing_index_isolated" + ) assert semantic_cache_mock.call_args_list[1].kwargs["filterable_fields"] == [ RedisSemanticCache._cache_key_filterable_field() ] @@ -233,7 +236,10 @@ def test_redis_semantic_cache_overwrites_stale_isolated_index(monkeypatch): ) assert redis_semantic_cache.llmcache is fallback_cache_mock - assert semantic_cache_mock.call_args_list[2].kwargs["name"] == "existing_index_isolated" + assert ( + semantic_cache_mock.call_args_list[2].kwargs["name"] + == "existing_index_isolated" + ) assert semantic_cache_mock.call_args_list[2].kwargs["overwrite"] is True assert semantic_cache_mock.call_args_list[2].kwargs["filterable_fields"] == [ RedisSemanticCache._cache_key_filterable_field() @@ -357,7 +363,9 @@ async def test_redis_semantic_cache_async_get_cache(monkeypatch): ] redis_semantic_cache.llmcache.acheck = AsyncMock(return_value=mock_result) - redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) with patch.object( redis_semantic_cache, @@ -405,7 +413,9 @@ async def test_redis_semantic_cache_async_get_cache_rejects_unscoped_hit(monkeyp } ] ) - redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) with patch.object( redis_semantic_cache, @@ -437,7 +447,9 @@ async def test_redis_semantic_cache_async_set_cache_stores_cache_key_filter( redis_semantic_cache = RedisSemanticCache(similarity_threshold=0.8) redis_semantic_cache.llmcache.astore = AsyncMock() - redis_semantic_cache._get_async_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) + redis_semantic_cache._get_async_embedding = AsyncMock( + return_value=[0.1, 0.2, 0.3] + ) await redis_semantic_cache.async_set_cache( key="test_key", @@ -597,7 +609,9 @@ def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None assert ( - RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_image", "image_url": "https://example.com"}]) + RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "input_image", "image_url": "https://example.com"}] + ) is None ) @@ -605,7 +619,9 @@ def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - prompt = RedisSemanticCache._get_prompt_from_kwargs(input={"text": " ", "input_text": "fallback prompt"}) + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input={"text": " ", "input_text": "fallback prompt"} + ) assert prompt == "fallback prompt" @@ -851,7 +867,9 @@ def test_redis_get_embedding_falls_back_to_direct(monkeypatch): fake_proxy.llm_model_list = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_proxy) - with patch("litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]}) as direct_embed: + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]} + ) as direct_embed: vec = cache._get_embedding("hello") assert vec == [0.1, 0.2] @@ -987,7 +1005,9 @@ def test_redis_sync_set_cache_passes_precomputed_vector(): cache = RedisSemanticCache.__new__(RedisSemanticCache) cache.llmcache = MagicMock() - cache._get_cache_filters = MagicMock(return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}) + cache._get_cache_filters = MagicMock( + return_value={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"} + ) cache._get_ttl = MagicMock(return_value=None) cache._get_embedding = MagicMock(return_value=[0.1, 0.2, 0.3]) @@ -1024,7 +1044,9 @@ def test_redis_sync_get_cache_passes_precomputed_vector(): ) cache._get_embedding = MagicMock(return_value=[0.1, 0.2, 0.3]) - with patch.object(cache, "_get_cache_key_filter_expression", return_value="cache-key-filter"): + with patch.object( + cache, "_get_cache_key_filter_expression", return_value="cache-key-filter" + ): result = cache.get_cache( key="test_key", messages=[{"content": "What is the capital of France?"}], @@ -1138,7 +1160,9 @@ def test_redis_get_embedding_truncates_direct_path_with_explicit_limit(monkeypat fake_proxy.llm_model_list = None monkeypatch.setitem(sys.modules, "litellm.proxy.proxy_server", fake_proxy) - with patch("litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]}) as direct_embed: + with patch( + "litellm.embedding", return_value={"data": [{"embedding": [0.1, 0.2]}]} + ) as direct_embed: cache._get_embedding(LONG_PROMPT) sent_input = direct_embed.call_args.kwargs["input"] @@ -1185,7 +1209,9 @@ def test_redis_init_defers_redisvl_construction(monkeypatch): def test_redis_failed_llmcache_build_is_not_memoized(monkeypatch): built_cache = MagicMock() - semantic_cache_mock = MagicMock(side_effect=[ConnectionError("redis down"), built_cache]) + semantic_cache_mock = MagicMock( + side_effect=[ConnectionError("redis down"), built_cache] + ) custom_vectorizer_mock = MagicMock() with _fake_redisvl_modules(semantic_cache_mock, custom_vectorizer_mock): From 6231a68f0ba634e43c4c0d9a03f378f584f77ab6 Mon Sep 17 00:00:00 2001 From: kerry Date: Tue, 6 Oct 2026 23:33:33 +0000 Subject: [PATCH 23/29] ci(integration): drop the caching group wiring that main already carries Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/integration/_support/manifest.py | 1 - tests/integration/run.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index a3b66c00e787..fb17ebb38640 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -11,7 +11,6 @@ "providers", "streaming", "messages_endpoint", - "caching", "translation", "configuration", "mcp", diff --git a/tests/integration/run.py b/tests/integration/run.py index 93772f2d8d89..df408cfc809a 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -15,7 +15,7 @@ "management": ("management", "authorization", "configuration"), "accounting": ("pricing", "spend"), "database": ("database",), - "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation", "caching"), + "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation"), "extensions": ("observability", "compatibility"), "mcp": ("mcp",), "sdk": ("sdk",), From c8ae23c4a0c03955389b01eb1d7470438d9b973a Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 00:07:01 +0000 Subject: [PATCH 24/29] refactor(caching): recurse into tool_result content in the semantic cache prompt helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../prompt_templates/common_utils.py | 31 ++++++------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 20dc2726de9e..15f1945b7920 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -197,12 +197,10 @@ def get_semantic_cache_prompt_from_messages(messages: Sequence[Mapping[str, obje The text a semantic cache embeds for a request: `get_str_from_messages` plus the text inside Messages API `tool_result` blocks, so a tool turn does not embed identically to the turn before it """ - return "".join(_semantic_cache_message_text(message) for message in messages) - - -def _semantic_cache_message_text(message: Mapping[str, object]) -> str: - return _semantic_cache_content_text(message.get("content")) + extract_search_results_text( - message.get("search_results") + return "".join( + _semantic_cache_content_text(message.get("content")) + + extract_search_results_text(message.get("search_results")) + for message in messages ) @@ -211,24 +209,13 @@ def _semantic_cache_content_text(content: object) -> str: return content if not isinstance(content, list): return "" - return "".join(_semantic_cache_block_text(block) for block in content) + return "".join(_semantic_cache_block_text(block) for block in content if isinstance(block, Mapping)) -def _semantic_cache_block_text(block: object) -> str: - if not isinstance(block, Mapping): - return "" - if block.get("type") != "tool_result": - return _text_field(block) - result: Final = block.get("content") - if isinstance(result, str): - return result - if isinstance(result, list): - return "".join(_text_field(inner) for inner in result) - return "" - - -def _text_field(block: object) -> str: - text: Final = block.get("text") if isinstance(block, Mapping) else None +def _semantic_cache_block_text(block: Mapping[str, object]) -> str: + if block.get("type") == "tool_result": + return _semantic_cache_content_text(block.get("content")) + text: Final = block.get("text") return text if isinstance(text, str) else "" From 0dc9c914bcab10320d9e2ae9bf21cb0bede3b31d Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 00:29:11 +0000 Subject: [PATCH 25/29] test(integration): check the max_messages cap on the shared exact-cache proxy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../caching/test_cache_max_messages.py | 409 +++--------------- 1 file changed, 69 insertions(+), 340 deletions(-) diff --git a/tests/integration/caching/test_cache_max_messages.py b/tests/integration/caching/test_cache_max_messages.py index 33bd0c09b4c9..89401b7c4ec7 100644 --- a/tests/integration/caching/test_cache_max_messages.py +++ b/tests/integration/caching/test_cache_max_messages.py @@ -1,274 +1,25 @@ -import hashlib import json -import math -import threading +import os import uuid -from collections.abc import Callable, Iterator, Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path +from collections.abc import Callable from typing import Final -from urllib.parse import urlsplit import pytest -import yaml -from integration._support.client import Gateway, eventually, gateway_from_environment -from integration._support.process import owned_proxy -from integration._support.wire import Reply, Request, wire_server -from pydantic import JsonValue, TypeAdapter +from integration._support.anthropic_sse import message_json +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.provider import SharedProvider +from integration._support.wire import Reply +from pydantic import JsonValue +from redis import Redis -_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) -_VECTOR_SIZE: Final = 16 -_CHAT_MODEL: Final = "capped-chat" -_CLAUDE_MODEL: Final = "capped-claude" -_EMBEDDING_MODEL: Final = "capped-embedder" -_COLLECTION: Final = "cache-max-messages" +_Turns = Callable[[str], tuple[list[JsonValue], list[JsonValue]]] -def _vector(text: str) -> tuple[float, ...]: - digest: Final = hashlib.sha256(text.encode()).digest() - return tuple((byte - 127.5) / 127.5 for byte in digest[:_VECTOR_SIZE]) - - -def _cosine(left: Sequence[float], right: Sequence[float]) -> float: - dot: Final = sum(a * b for a, b in zip(left, right, strict=True)) - return dot / (math.sqrt(sum(a * a for a in left)) * math.sqrt(sum(b * b for b in right))) - - -def _answer() -> str: - return f"answer-{uuid.uuid4().hex[:16]}" - - -@dataclass(slots=True) -class _Peer: - """Embeddings, chat, Responses, Anthropic Messages and a Qdrant collection on one owned socket""" - - lock: threading.Lock = field(default_factory=threading.Lock) - points: list[Mapping[str, JsonValue]] = field(default_factory=list) # mutable-ok: the Qdrant collection - embedded: list[str] = field(default_factory=list) # mutable-ok: every prompt the proxy embedded - answered: list[str] = field(default_factory=list) # mutable-ok: every completion the provider served - - def stored(self) -> int: - with self.lock: - return len(self.points) - - def respond(self, request: Request) -> Reply: - path: Final = urlsplit(request.target).path - body: Final = _JSON_OBJECT.validate_json(request.body) if request.body else {} - collection: Final = f"/qdrant/collections/{_COLLECTION}" - if request.method == "GET" and path == f"{collection}/exists": - return self._json({"result": {"exists": False}, "status": "ok"}) - if request.method in {"GET", "PUT"} and path in {collection, f"{collection}/index"}: - return self._json({"result": True, "status": "ok"}) - if request.method == "PUT" and path == f"{collection}/points": - with self.lock: - self.points.extend(_JSON_OBJECT.validate_python(point) for point in _list(body["points"])) - return self._json({"result": {"status": "completed"}, "status": "ok"}) - if request.method == "POST" and path == f"{collection}/points/search": - return self._json({"result": self._search(body), "status": "ok"}) - if request.method == "GET" and path == "/v1/models": - return self._json({"object": "list", "data": []}) - if request.method == "POST" and path == "/v1/embeddings": - text: Final = str(body["input"]) - with self.lock: - self.embedded.append(text) - return self._json( - { - "object": "list", - "model": "text-embedding-3-small", - "data": [{"object": "embedding", "index": 0, "embedding": list(_vector(text))}], - "usage": {"prompt_tokens": 4, "total_tokens": 4}, - } - ) - if request.method == "POST" and path == "/v1/chat/completions": - return self._json(self._chat_reply(_answer())) - if request.method == "POST" and path == "/v1/responses": - return self._json(self._responses_reply(_answer())) - if request.method == "POST" and path == "/v1/messages": - return self._json(self._messages_reply(_answer())) - raise AssertionError(f"unexpected peer request {request.method} {request.target}") - - def _search(self, body: Mapping[str, JsonValue]) -> list[JsonValue]: - query: Final = [float(str(value)) for value in _list(body["vector"])] - key: Final = _JSON_OBJECT.validate_python( - _JSON_OBJECT.validate_python(_list(_JSON_OBJECT.validate_python(body["filter"])["must"])[0])["match"] - )["value"] - with self.lock: - scoped: Final = [ - point - for point in self.points - if _JSON_OBJECT.validate_python(point["payload"])["litellm_cache_key"] == key - ] - ranked: Final = sorted( - ( - { - "id": point["id"], - "score": _cosine(query, [float(str(value)) for value in _list(point["vector"])]), - "payload": point["payload"], - } - for point in scoped - ), - key=lambda hit: -float(str(hit["score"])), - ) - return list(ranked[:1]) - - def _chat_reply(self, answer: str) -> Mapping[str, JsonValue]: - with self.lock: - self.answered.append(answer) - return { - "id": f"chatcmpl-{uuid.uuid4().hex}", - "object": "chat.completion", - "created": 1789788253, - "model": "gpt-5.4-mini", - "choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12}, - } - - def _responses_reply(self, answer: str) -> Mapping[str, JsonValue]: - with self.lock: - self.answered.append(answer) - identity: Final = uuid.uuid4().hex - return { - "id": f"resp_{identity}", - "object": "response", - "created_at": 1789788253, - "status": "completed", - "model": "gpt-5.4-mini", - "output": [ - { - "id": f"msg_{identity}", - "type": "message", - "role": "assistant", - "status": "completed", - "content": [{"type": "output_text", "text": answer, "annotations": []}], - } - ], - "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, - } - - def _messages_reply(self, answer: str) -> Mapping[str, JsonValue]: - with self.lock: - self.answered.append(answer) - return { - "id": f"msg_{uuid.uuid4().hex}", - "type": "message", - "role": "assistant", - "model": "claude-sonnet-5-5", - "content": [{"type": "text", "text": answer}], - "stop_reason": "end_turn", - "stop_sequence": None, - "usage": {"input_tokens": 10, "output_tokens": 2}, - } - - @staticmethod - def _json(value: Mapping[str, JsonValue]) -> Reply: - return Reply(body=json.dumps(value).encode()) - - -def _list(value: JsonValue) -> list[JsonValue]: - assert isinstance(value, list), value - return value - - -@dataclass(frozen=True, slots=True) -class _Proxy: - gateway: Gateway - peer: _Peer - - def reply(self, route: str, conversation: list[JsonValue]) -> str: - """The answer text for one request, as the client sees it""" - model: Final = _CLAUDE_MODEL if route == "/v1/messages" else _CHAT_MODEL - body: Final[dict[str, JsonValue]] = ( - {"model": model, "input": conversation} - if route == "/v1/responses" - else {"model": model, "max_tokens": 16, "messages": conversation} - ) - response: Final = self.gateway.request("POST", route, body) - assert response.status_code == 200, response.text - payload: Final = _JSON_OBJECT.validate_json(response.content) - if route == "/v1/messages": - return str(_JSON_OBJECT.validate_python(_list(payload["content"])[0])["text"]) - if route == "/v1/responses": - message: Final = _JSON_OBJECT.validate_python(_list(payload["output"])[0]) - return str(_JSON_OBJECT.validate_python(_list(message["content"])[0])["text"]) - choice: Final = _JSON_OBJECT.validate_python(_list(payload["choices"])[0]) - return str(_JSON_OBJECT.validate_python(choice["message"])["content"]) - - def miss(self, route: str, conversation: list[JsonValue]) -> str: - answered_before: Final = len(self.peer.answered) - answer: Final = self.reply(route, conversation) - assert len(self.peer.answered) == answered_before + 1, f"{answer} was served from the cache" - return answer - - def hit(self, route: str, conversation: list[JsonValue]) -> str: - """Repeats the request until the cache serves it, since the store after a miss is asynchronous""" - - def attempt() -> tuple[str, bool]: - answered_before: Final = len(self.peer.answered) - answer: Final = self.reply(route, conversation) - return answer, len(self.peer.answered) == answered_before - - return eventually(attempt, lambda outcome: outcome[1])[0] - - -def _proxy(tmp_path_factory: pytest.TempPathFactory, cache_params: Mapping[str, JsonValue]) -> Iterator[_Proxy]: - peer: Final = _Peer() - directory: Final = tmp_path_factory.mktemp("cache-max-messages") - with gateway_from_environment() as gateway, wire_server(peer.respond) as wire: - config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) - config["model_list"] = [ - { - "model_name": _CHAT_MODEL, - "litellm_params": {"model": "openai/gpt-5.4-mini", "api_base": f"{wire.url}/v1", "api_key": "k"}, - }, - { - "model_name": _CLAUDE_MODEL, - "litellm_params": {"model": "anthropic/claude-sonnet-5-5", "api_base": wire.url, "api_key": "k"}, - }, - { - "model_name": _EMBEDDING_MODEL, - "litellm_params": { - "model": "openai/text-embedding-3-small", - "api_base": f"{wire.url}/v1", - "api_key": "k", - }, - }, - ] - config["litellm_settings"]["cache_params"] = { - **cache_params, - **({"qdrant_api_base": f"{wire.url}/qdrant"} if cache_params["type"] == "qdrant-semantic" else {}), - } - path: Final = directory / "cache_max_messages.yaml" - path.write_text(yaml.safe_dump(config)) - with owned_proxy(gateway, directory, {}, config=path) as candidate: - yield _Proxy(candidate, peer) - - -@pytest.fixture(scope="module") -def qdrant_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Proxy]: - """Qdrant semantic cache with the default max_messages of 4""" - yield from _proxy( - tmp_path_factory, - { - "type": "qdrant-semantic", - "qdrant_collection_name": _COLLECTION, - "qdrant_semantic_cache_embedding_model": _EMBEDDING_MODEL, - "qdrant_semantic_cache_vector_size": _VECTOR_SIZE, - "qdrant_quantization_config": "binary", - "similarity_threshold": 0.99, - }, - ) - - -@pytest.fixture(scope="module") -def exact_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Proxy]: - """Exact cache with max_messages lowered to 2 in cache_params""" - yield from _proxy(tmp_path_factory, {"type": "local", "max_messages": 2}) - - -def _claude_code_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: - """Turns 1 to 3 of a Claude Code session on /v1/messages: 1, 3 and 5 messages""" +def _claude_code_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of a Claude Code session on /v1/messages: 1 and 5 messages""" first: Final[list[JsonValue]] = [{"role": "user", "content": task}] - second: Final[list[JsonValue]] = [ + third: Final[list[JsonValue]] = [ *first, { "role": "assistant", @@ -278,42 +29,28 @@ def _claude_code_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], lis "role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "calc.py test_calc.py"}], }, - ] - third: Final[list[JsonValue]] = [ - *second, { "role": "assistant", "content": [{"type": "tool_use", "id": "toolu_2", "name": "Read", "input": {"file_path": "calc.py"}}], }, { "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "toolu_2", - "content": [{"type": "text", "text": "def add(a, b): return a - b"}], - } - ], + "content": [{"type": "tool_result", "tool_use_id": "toolu_2", "content": "def add(a, b): return a - b"}], }, ] - return first, second, third + return first, third -def _agent_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: - """Turns 1 to 3 of an OpenAI tool loop on /v1/chat/completions: 2, 4 and 6 messages""" +def _agent_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of an OpenAI tool loop on /v1/chat/completions: 2 and 6 messages""" def call(call_id: str, path: str) -> list[JsonValue]: + function: Final[JsonValue] = {"name": "write_file", "arguments": json.dumps({"path": path})} return [ { "role": "assistant", "content": None, - "tool_calls": [ - { - "id": call_id, - "type": "function", - "function": {"name": "write_file", "arguments": json.dumps({"path": path})}, - } - ], + "tool_calls": [{"id": call_id, "type": "function", "function": function}], }, {"role": "tool", "tool_call_id": call_id, "content": f"wrote {path}"}, ] @@ -322,13 +59,11 @@ def call(call_id: str, path: str) -> list[JsonValue]: {"role": "system", "content": "You are a coding agent"}, {"role": "user", "content": task}, ] - second: Final[list[JsonValue]] = [*first, *call("call_1", "a.yaml")] - third: Final[list[JsonValue]] = [*second, *call("call_2", "b.yaml")] - return first, second, third + return first, [*first, *call("call_1", "a.yaml"), *call("call_2", "b.yaml")] -def _responses_turns(task: str) -> tuple[list[JsonValue], list[JsonValue], list[JsonValue]]: - """Turns 1 to 3 of an agent on /v1/responses: 1, 3 and 5 input items""" +def _responses_turns(task: str) -> tuple[list[JsonValue], list[JsonValue]]: + """Turns 1 and 3 of an agent on /v1/responses: 1 and 5 input items""" def call(call_id: str, path: str) -> list[JsonValue]: return [ @@ -342,74 +77,68 @@ def call(call_id: str, path: str) -> list[JsonValue]: ] first: Final[list[JsonValue]] = [{"role": "user", "content": task}] - second: Final[list[JsonValue]] = [*first, *call("call_1", "a.yaml")] - third: Final[list[JsonValue]] = [*second, *call("call_2", "b.yaml")] - return first, second, third + return first, [*first, *call("call_1", "a.yaml"), *call("call_2", "b.yaml")] -def _assert_turns_are_cached_up_to_four_messages( - proxy: _Proxy, route: str, turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]] -) -> None: - first, second, third = turns(f"update the config {uuid.uuid4().hex}") - stored_before: Final = proxy.peer.stored() +def _body(path: str, model: str, conversation: list[JsonValue]) -> dict[str, JsonValue]: + if path == "/v1/responses": + return {"model": model, "input": conversation} + return {"model": model, "max_tokens": 16, "messages": conversation} - short_answers: Final = (proxy.miss(route, first), proxy.miss(route, second)) - repeated: Final = (proxy.hit(route, first), proxy.hit(route, second)) - long_answers: Final = (proxy.miss(route, third), proxy.miss(route, third)) - proxy.hit(route, first) - assert repeated == short_answers, f"a repeated short turn got another turn's answer: {short_answers} {repeated}" - assert long_answers[0] != long_answers[1], "a turn past max_messages was served from the cache" - assert all("b.yaml" not in text and "def add" not in text for text in proxy.peer.embedded), ( - "a turn past max_messages was embedded" - ) - assert proxy.peer.stored() == stored_before + 2, "a turn past max_messages was written to the cache" +def _reply(path: str, text: str) -> Reply: + identity: Final = f"id_{uuid.uuid4().hex}" + if path == "/v1/messages": + return Reply(body=message_json(identity, "claude-sonnet-5-5", text)) + if path == "/v1/responses": + return responses_reply(identity, "gpt-5.6-sol", text, stream=False) + return chat_reply(identity, "gpt-5.4", text, stream=False) -@pytest.mark.parametrize( - ("route", "turns"), - [ - pytest.param("/v1/messages", _claude_code_turns, id="messages"), - pytest.param("/v1/chat/completions", _agent_turns, id="chat-completions"), - ], -) -def test_agent_turns_are_cached_up_to_four_messages_and_bypass_the_cache_past_it( - qdrant_proxy: _Proxy, - route: str, - turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]], -) -> None: - _assert_turns_are_cached_up_to_four_messages(qdrant_proxy, route, turns) +def _answer(path: str, payload: dict[str, JsonValue]) -> str: + if path == "/v1/messages": + return string_value(_first(payload["content"])["text"]) + if path == "/v1/responses": + return string_value(_first(_first(payload["output"])["content"])["text"]) + return string_value(object_value(_first(payload["choices"])["message"])["content"]) -def test_tool_result_text_tells_a_tool_turn_apart_from_the_turn_before_it(qdrant_proxy: _Proxy) -> None: - task: Final = f"list the files {uuid.uuid4().hex}" - first, second, _ = _claude_code_turns(task) +def _first(value: JsonValue) -> dict[str, JsonValue]: + assert isinstance(value, list), value + return object_value(value[0]) - qdrant_proxy.miss("/v1/messages", first) - qdrant_proxy.miss("/v1/messages", second) - assert task in qdrant_proxy.peer.embedded[-1], qdrant_proxy.peer.embedded[-1] - assert "calc.py test_calc.py" in qdrant_proxy.peer.embedded[-1], qdrant_proxy.peer.embedded[-1] +def _cached_responses(redis: Redis) -> frozenset[bytes]: + digests: Final = tuple(key for key in redis.scan_iter() if len(key) == 64) + return frozenset(key for key in digests if b'"response"' in (redis.get(key) or b"")) @pytest.mark.parametrize( - ("route", "turns"), + ("path", "model", "turns"), [ - pytest.param("/v1/messages", _claude_code_turns, id="messages"), - pytest.param("/v1/chat/completions", _agent_turns, id="chat-completions"), - pytest.param("/v1/responses", _responses_turns, id="responses"), + pytest.param("/v1/messages", "anthropic/claude-sonnet-5-5", _claude_code_turns, id="messages"), + pytest.param("/v1/chat/completions", "openai/gpt-5.4", _agent_turns, id="chat-completions"), + pytest.param("/v1/responses", "openai/responses/gpt-5.6-sol", _responses_turns, id="responses"), ], ) -def test_exact_cache_honours_max_messages_from_cache_params( - exact_proxy: _Proxy, - route: str, - turns: Callable[[str], tuple[list[JsonValue], list[JsonValue], list[JsonValue]]], +def test_cache_serves_a_turn_under_max_messages_and_skips_one_past_it( + gateway: Gateway, provider: SharedProvider, path: str, model: str, turns: _Turns ) -> None: - first, second, _ = turns(f"what is {uuid.uuid4().hex}") - - answer: Final = exact_proxy.miss(route, first) - repeated: Final = exact_proxy.hit(route, first) - long_answers: Final = (exact_proxy.miss(route, second), exact_proxy.miss(route, second)) + under_cap, past_cap = turns(f"update the config {uuid.uuid4().hex}") + provider.expect(_reply(path, "first answer"), _reply(path, "second answer"), _reply(path, "third answer")) + + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as redis: + cached_before: Final = _cached_responses(redis) + gateway.post(path, _body(path, model, under_cap)) + eventually(lambda: _cached_responses(redis) - cached_before, lambda written: len(written) == 1) + repeated: Final = gateway.post(path, _body(path, model, under_cap)) + past_cap_twice: Final = ( + gateway.post(path, _body(path, model, past_cap)), + gateway.post(path, _body(path, model, past_cap)), + ) - assert repeated == answer - assert long_answers[0] != long_answers[1], "a turn past max_messages was served from a cache capped at 2" + assert _answer(path, repeated) == "first answer", "a repeated turn under max_messages was not served from the cache" + assert tuple(_answer(path, answer) for answer in past_cap_twice) == ("second answer", "third answer"), ( + "a turn past max_messages was served from the cache" + ) + assert len(provider.received()) == 3 From f6e9be9e8132ef2f340b0c75640c746f3f4b7b65 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 00:44:45 +0000 Subject: [PATCH 26/29] fix(caching): read list-form function_call_output text in semantic cache prompts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/crates/cache/src/semantic.rs | 4 ++++ litellm-rust/crates/cache/tests/semantic.rs | 9 +++++++++ litellm/caching/redis_semantic_cache.py | 5 +++++ tests/unit/caching/test_redis_semantic_cache.py | 14 ++++++++++++++ 4 files changed, 32 insertions(+) diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 5aafb333b391..6200645e5557 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -184,6 +184,10 @@ fn collect_input_text(value: &Value, parts: &mut Vec) { collect_input_text(content, parts); return; } + if let Some(output) = map.get("output").filter(|output| output.is_array()) { + collect_input_text(output, parts); + return; + } for key in ["text", "output", "input_text", "output_text"] { if let Some(Value::String(text)) = map.get(key) && push_trimmed(text, parts) diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index e0a39bb9d5a6..76af1863e1fc 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -191,6 +191,15 @@ fn prompt_from_messages_reads_messages_only( ])), Some("model dump prompt\ndict prompt\ninline prompt"), )] +#[case::function_call_output_blocks( + None, + Some(json!([ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\": \"a\"}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]}, + ])), + Some("update the config\nwrote a"), +)] #[case::object_content( None, Some(json!({"content": [{"text": "object content prompt"}]})), diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index e39c2074739f..8820de43b465 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -296,6 +296,11 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - cls._collect_responses_input_text(content, prompt_parts) return + output = value.get("output") + if isinstance(output, list): + cls._collect_responses_input_text(output, prompt_parts) + return + for text_key in ("text", "output", "input_text", "output_text"): text_value = value.get(text_key) if isinstance(text_value, str): diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 9077cded532f..695a4d1da031 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -568,6 +568,20 @@ def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): ) +def test_redis_semantic_cache_prompt_extraction_reads_function_call_output_blocks(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": '{"path": "a"}'}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "wrote a"}]}, + ] + ) + + assert prompt == "update the config\nwrote a" + + def test_redis_semantic_cache_prompt_extraction_prefers_messages(): from litellm.caching.redis_semantic_cache import RedisSemanticCache From 0665296bf1d783583ffc7f494e111ef58fec2b18 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 16:34:03 +0000 Subject: [PATCH 27/29] refactor(caching): extract nested Responses input lookup to keep walker under complexity limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_semantic_cache.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 8820de43b465..058be61f1351 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -291,14 +291,9 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - return if isinstance(value, dict): - content = value.get("content") - if content is not None: - cls._collect_responses_input_text(content, prompt_parts) - return - - output = value.get("output") - if isinstance(output, list): - cls._collect_responses_input_text(output, prompt_parts) + nested: Final = cls._nested_response_input(value) + if nested is not None: + cls._collect_responses_input_text(nested, prompt_parts) return for text_key in ("text", "output", "input_text", "output_text"): @@ -323,6 +318,14 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - prompt_parts.append(stripped_text) return + @staticmethod + def _nested_response_input(value: dict[str, object]) -> object: + content: Final = value.get("content") + if content is not None: + return content + output: Final = value.get("output") + return output if isinstance(output, list) else None + @staticmethod def _coerce_response_input_value(value: object) -> object: model_dump: Final = getattr(value, "model_dump", None) From ba93ca578d87c5f2ecb9584e5139a3179a014cb9 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 17:22:01 +0000 Subject: [PATCH 28/29] Revert "refactor(caching): extract nested Responses input lookup to keep walker under complexity limit" This reverts commit 0665296bf1d783583ffc7f494e111ef58fec2b18. --- litellm/caching/redis_semantic_cache.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 058be61f1351..8820de43b465 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -291,9 +291,14 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - return if isinstance(value, dict): - nested: Final = cls._nested_response_input(value) - if nested is not None: - cls._collect_responses_input_text(nested, prompt_parts) + content = value.get("content") + if content is not None: + cls._collect_responses_input_text(content, prompt_parts) + return + + output = value.get("output") + if isinstance(output, list): + cls._collect_responses_input_text(output, prompt_parts) return for text_key in ("text", "output", "input_text", "output_text"): @@ -318,14 +323,6 @@ def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) - prompt_parts.append(stripped_text) return - @staticmethod - def _nested_response_input(value: dict[str, object]) -> object: - content: Final = value.get("content") - if content is not None: - return content - output: Final = value.get("output") - return output if isinstance(output, list) else None - @staticmethod def _coerce_response_input_value(value: object) -> object: model_dump: Final = getattr(value, "model_dump", None) From 580a9702b0de9a7983f415c3f4c185a1a82c4aaa Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 7 Oct 2026 17:22:45 +0000 Subject: [PATCH 29/29] style(caching): suppress C901 on the Responses input walker instead of splitting it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/caching/redis_semantic_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 8820de43b465..8f99d76ba46e 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -274,7 +274,7 @@ def _get_prompt_from_kwargs(cls, **kwargs) -> str | None: return prompt or None @classmethod - def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: + def _collect_responses_input_text(cls, value: object, prompt_parts: list[str]) -> None: # noqa: C901 # one branch per Responses input shape value = cls._coerce_response_input_value(value) if value is None: return