Skip to content

fix: support Responses input in Redis semantic cache #29581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Sameerlite merged 7 commits into from
Jun 8, 2026
Merged

fix: support Responses input in Redis semantic cache #29581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
05d9657
fix: support responses input in redis semantic cache
balcsida May 19, 2026
af948a1
test: cover redis semantic prompt extraction
balcsida May 19, 2026
dcad0e0
test: handle blank redis semantic text fallbacks
balcsida May 19, 2026
d775953
chore: remove async cache dead statement
balcsida May 23, 2026
4da8111
test: cover redis semantic cache miss paths
balcsida May 23, 2026
37406e7
fix: filter sensitive cache lookup kwargs
balcsida May 23, 2026
8777fe7
chore: rerun ci after huggingface rate limit
balcsida Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions litellm/caching/caching.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,34 @@ def _get_cache_logic(
return cached_response
return cached_result

@staticmethod
def _get_safe_cache_lookup_kwargs(kwargs: Dict[str, Any]) -> Dict[str, Any]:
cache_lookup_kwargs: Dict[str, Any] = {}
for prompt_kwarg in ("messages", "input"):
if prompt_kwarg in kwargs:
cache_lookup_kwargs[prompt_kwarg] = kwargs[prompt_kwarg]

if isinstance(kwargs.get("metadata"), dict):
cache_lookup_kwargs["metadata"] = {}

return cache_lookup_kwargs

@staticmethod
def _update_metadata_from_cache_lookup_kwargs(
original_kwargs: Dict[str, Any], cache_lookup_kwargs: Dict[str, Any]
) -> None:
original_metadata = original_kwargs.get("metadata")
cache_lookup_metadata = cache_lookup_kwargs.get("metadata")
if not isinstance(original_metadata, dict) or not isinstance(
cache_lookup_metadata, dict
):
return

if "semantic-similarity" in cache_lookup_metadata:
original_metadata["semantic-similarity"] = cache_lookup_metadata[
"semantic-similarity"
]

def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
"""
Retrieves the cached result for the given arguments.
Expand All @@ -511,7 +539,6 @@ def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
try: # never block execution
if self.should_use_cache(**kwargs) is not True:
return
messages = kwargs.get("messages", [])
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
Expand All @@ -523,12 +550,19 @@ def get_cache(self, dynamic_cache_object: Optional[BaseCache] = None, **kwargs):
or cache_control_args.get("s-max-age")
or float("inf")
)
cache_lookup_kwargs = self._get_safe_cache_lookup_kwargs(kwargs)
if dynamic_cache_object is not None:
cached_result = dynamic_cache_object.get_cache(
cache_key, messages=messages
cache_key, **cache_lookup_kwargs
)
else:
cached_result = self.cache.get_cache(cache_key, messages=messages)
cached_result = self.cache.get_cache(
cache_key, **cache_lookup_kwargs
)
self._update_metadata_from_cache_lookup_kwargs(
original_kwargs=kwargs,
cache_lookup_kwargs=cache_lookup_kwargs,
)
return self._get_cache_logic(
cached_result=cached_result, max_age=max_age
)
Expand All @@ -549,7 +583,6 @@ async def async_get_cache(
if self.should_use_cache(**kwargs) is not True:
return

kwargs.get("messages", [])
if "cache_key" in kwargs:
cache_key = kwargs["cache_key"]
else:
Expand Down
105 changes: 84 additions & 21 deletions litellm/caching/redis_semantic_cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,78 @@ def _get_ttl(self, **kwargs) -> Optional[int]:
ttl = int(ttl)
return ttl

@classmethod
def _get_prompt_from_kwargs(cls, **kwargs) -> Optional[str]:
"""
Extract a semantic-cache prompt from chat or Responses API request kwargs.
"""
messages = kwargs.get("messages")
if messages:
return get_str_from_messages(messages)

if "input" not in kwargs:
return None

prompt_parts: List[str] = []
cls._collect_responses_input_text(kwargs.get("input"), prompt_parts)
prompt = "\n".join(prompt_parts).strip()
return prompt or None

@classmethod
def _collect_responses_input_text(cls, value: Any, prompt_parts: List[str]) -> None:
value = cls._coerce_response_input_value(value)
if value is None:
return

if isinstance(value, str):
stripped_value = value.strip()
if stripped_value:
prompt_parts.append(stripped_value)
return

if isinstance(value, (list, tuple)):
for item in value:
cls._collect_responses_input_text(item, prompt_parts)
return

if isinstance(value, dict):
content = value.get("content")
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return

for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
return

content = getattr(value, "content", None)
if content is not None:
cls._collect_responses_input_text(content, prompt_parts)
return

for text_key in ("text", "output", "input_text", "output_text"):
text_value = getattr(value, text_key, None)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return

@staticmethod
def _coerce_response_input_value(value: Any) -> Any:
model_dump = getattr(value, "model_dump", None)
if callable(model_dump):
return model_dump()
dict_method = getattr(value, "dict", None)
if callable(dict_method):
return dict_method()
return value

def _get_embedding(self, prompt: str) -> List[float]:
"""
Generate an embedding vector for the given prompt using the configured embedding model.
Expand Down Expand Up @@ -278,13 +350,11 @@ def set_cache(self, key: str, value: Any, **kwargs) -> None:

value_str: Optional[str] = None
try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return

prompt = get_str_from_messages(messages)
value_str = str(value)

store_kwargs: Dict[str, Any] = {
Expand Down Expand Up @@ -315,14 +385,12 @@ def get_cache(self, key: str, **kwargs) -> Any:
print_verbose(f"Redis semantic-cache get_cache, kwargs: {kwargs}")

try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None

prompt = get_str_from_messages(messages)
# Check the cache for semantically similar prompts in this exact
# LiteLLM cache-key scope.
check_kwargs: Dict[str, Any] = {
Expand Down Expand Up @@ -428,13 +496,11 @@ async def async_set_cache(self, key: str, value: Any, **kwargs) -> None:
print_verbose(f"Async Redis semantic-cache set_cache, kwargs: {kwargs}")

try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic caching")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic caching")
return

prompt = get_str_from_messages(messages)
value_str = str(value)

# Generate embedding for the value (response) to cache
Expand Down Expand Up @@ -471,15 +537,12 @@ async def async_get_cache(self, key: str, **kwargs) -> Any:
print_verbose(f"Async Redis semantic-cache get_cache, kwargs: {kwargs}")

try:
# Extract the prompt from messages
messages = kwargs.get("messages", [])
if not messages:
print_verbose("No messages provided for semantic cache lookup")
prompt = self._get_prompt_from_kwargs(**kwargs)
if prompt is None:
print_verbose("No prompt provided for semantic cache lookup")
kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
return None

prompt = get_str_from_messages(messages)

# Generate embedding for the prompt
prompt_embedding = await self._get_async_embedding(prompt, **kwargs)

Expand Down
Loading
Loading