(refactor) caching use LLMCachingHandler for async_get_cache and set_cache

cookbook/litellm_router_load_test/test_loadtest_openai_client.py

@@ -10,7 +10,7 @@
 import asyncio
 from litellm import Router, Timeout
 import time
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 import litellm
 import openai
 

cookbook/litellm_router_load_test/test_loadtest_router_withs3_cache.py

@@ -10,7 +10,7 @@
 import asyncio
 from litellm import Router, Timeout
 import time
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 import litellm
 
 litellm.cache = Cache(

docs/my-website/docs/caching/all_caches.md

@@ -3,7 +3,7 @@ import TabItem from '@theme/TabItem';
 
 # Caching - In-Memory, Redis, s3, Redis Semantic Cache, Disk
 
-[**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm/caching.py)
+[**See Code**](https://github.com/BerriAI/litellm/blob/main/litellm.caching.caching.py)
 
 :::info
 
@@ -31,7 +31,7 @@ For the hosted version you can setup your own Redis DB here: https://app.redisla
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 litellm.cache = Cache(type="redis", host=<host>, port=<port>, password=<password>)
 
@@ -68,7 +68,7 @@ AWS_SECRET_ACCESS_KEY = "WOl*****"
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 # pass s3-bucket name
 litellm.cache = Cache(type="s3", s3_bucket_name="cache-bucket-litellm", s3_region_name="us-west-2")
@@ -101,7 +101,7 @@ For the hosted version you can setup your own Redis DB here: https://app.redisla
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 random_number = random.randint(
     1, 100000
@@ -155,7 +155,7 @@ To set up a Qdrant cluster locally follow: https://qdrant.tech/documentation/qui
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 random_number = random.randint(
     1, 100000
@@ -210,7 +210,7 @@ assert response1.id == response2.id
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache()
 
 # Make completion calls
@@ -246,7 +246,7 @@ Then you can use the disk cache as follows.
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache(type="disk")
 
 # Make completion calls
@@ -422,7 +422,7 @@ def custom_get_cache_key(*args, **kwargs):
 
 Set your function as litellm.cache.get_cache_key
 ```python
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 cache = Cache(type="redis", host=os.environ['REDIS_HOST'], port=os.environ['REDIS_PORT'], password=os.environ['REDIS_PASSWORD'])
 
@@ -434,7 +434,7 @@ litellm.cache = cache # set litellm.cache to your cache
 ## How to write custom add/get cache functions 
 ### 1. Init Cache 
 ```python
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 cache = Cache()
 ``` 
 

docs/my-website/docs/caching/caching_api.md

@@ -6,7 +6,7 @@ Use api.litellm.ai for caching `completion()` and `embedding()` responses
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache(type="hosted") # init cache to use api.litellm.ai
 
 # Make completion calls
@@ -31,7 +31,7 @@ response2 = completion(
 import time
 import litellm
 from litellm import completion, embedding
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache(type="hosted")
 
 start_time = time.time()
@@ -53,7 +53,7 @@ LiteLLM can cache your streamed responses for you
 import litellm
 import time
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 
 litellm.cache = Cache(type="hosted")
 

docs/my-website/docs/caching/local_caching.md

@@ -13,7 +13,7 @@ Keys in the cache are `model`, the following example will lead to a cache hit
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache()
 
 # Make completion calls
@@ -35,7 +35,7 @@ response2 = completion(
 Add custom key-value pairs to your cache. 
 
 ```python 
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 cache = Cache()
 
 cache.add_cache(cache_key="test-key", result="1234")
@@ -50,7 +50,7 @@ LiteLLM can cache your streamed responses for you
 ```python
 import litellm
 from litellm import completion
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache()
 
 # Make completion calls
@@ -77,7 +77,7 @@ Keys in the cache are `model`, the following example will lead to a cache hit
 import time
 import litellm
 from litellm import embedding
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 litellm.cache = Cache()
 
 start_time = time.time()

docs/my-website/docs/proxy/caching.md

@@ -49,13 +49,13 @@ litellm_settings:
   cache: true 
   cache_params:        # set cache params for redis
     type: redis
-    namespace: "litellm_caching"
+    namespace: "litellm.caching.caching"
 ```
 
 and keys will be stored like:
 
 ```
-litellm_caching:<hash>
+litellm.caching.caching:<hash>
 ```
 
 #### Redis Cluster 

docs/my-website/docs/proxy/configs.md

@@ -645,7 +645,7 @@ litellm_settings:
     host: "localhost"  # The host address for the Redis cache. Required if type is "redis".
     port: 6379  # The port number for the Redis cache. Required if type is "redis".
     password: "your_password"  # The password for the Redis cache. Required if type is "redis".
-    namespace: "litellm_caching" # namespace for redis cache
+    namespace: "litellm.caching.caching" # namespace for redis cache
 
     # Optional - Redis Cluster Settings
     redis_startup_nodes: [{"host": "127.0.0.1", "port": "7001"}] 

docs/my-website/docs/proxy/guardrails/custom_guardrail.md

@@ -25,7 +25,7 @@ from typing import Any, Dict, List, Literal, Optional, Union
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata

enterprise/enterprise_callbacks/generic_api_callback.py

@@ -6,7 +6,7 @@
 import requests
 
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 
 from typing import Literal, Union, Optional
 

enterprise/enterprise_hooks/aporia_ai.py

@@ -13,7 +13,7 @@
 )  # Adds the parent directory to the system path
 from typing import Optional, Literal, Union, Any
 import litellm, traceback, sys, uuid
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from fastapi import HTTPException

enterprise/enterprise_hooks/banned_keywords.py

@@ -9,7 +9,7 @@
 
 from typing import Optional, Literal
 import litellm
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger

enterprise/enterprise_hooks/blocked_user_list.py

@@ -10,7 +10,7 @@
 from typing import Optional, Literal
 import litellm
 from litellm.proxy.utils import PrismaClient
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth, LiteLLM_EndUserTable
 from litellm.integrations.custom_logger import CustomLogger
 from litellm._logging import verbose_proxy_logger

enterprise/enterprise_hooks/google_text_moderation.py

@@ -9,7 +9,7 @@
 
 from typing import Optional, Literal, Union
 import litellm, traceback, sys, uuid
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException

enterprise/enterprise_hooks/llama_guard.py

@@ -15,7 +15,7 @@
 )  # Adds the parent directory to the system path
 from typing import Optional, Literal, Union
 import litellm, traceback, sys, uuid
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException

enterprise/enterprise_hooks/llm_guard.py

@@ -13,7 +13,7 @@
 import sys
 import uuid
 import os
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException

enterprise/enterprise_hooks/openai_moderation.py

@@ -12,7 +12,7 @@
 )  # Adds the parent directory to the system path
 from typing import Optional, Literal, Union
 import litellm, traceback, sys, uuid
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.integrations.custom_logger import CustomLogger
 from fastapi import HTTPException

enterprise/enterprise_hooks/secret_detection.py

@@ -12,7 +12,7 @@
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
 from typing import Optional
-from litellm.caching import DualCache
+from litellm.caching.caching import DualCache
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm._logging import verbose_proxy_logger
 import tempfile

litellm/__init__.py

@@ -7,7 +7,7 @@
 import os
 from typing import Callable, List, Optional, Dict, Union, Any, Literal, get_args
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
-from litellm.caching import Cache
+from litellm.caching.caching import Cache
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,

litellm/batch_completion/main.py

@@ -2,7 +2,6 @@
 from typing import List, Optional
 
 import litellm
-from litellm import completion
 from litellm._logging import print_verbose
 from litellm.utils import get_optional_params
 
@@ -108,7 +107,7 @@ def chunks(lst, n):
                     if "kwargs" in kwargs_modified:
                         original_kwargs = kwargs_modified.pop("kwargs")
                     future = executor.submit(
-                        completion, **kwargs_modified, **original_kwargs
+                        litellm.completion, **kwargs_modified, **original_kwargs
                     )
                     completions.append(future)
 
@@ -156,7 +155,7 @@ def batch_completion_models(*args, **kwargs):
         with ThreadPoolExecutor(max_workers=len(models)) as executor:
             for model in models:
                 futures[model] = executor.submit(
-                    completion, *args, model=model, **kwargs
+                    litellm.completion, *args, model=model, **kwargs
                 )
 
             for model, future in sorted(
@@ -178,7 +177,9 @@ def batch_completion_models(*args, **kwargs):
                     ):  # don't override deployment values e.g. model name, api base, etc.
                         deployment[key] = kwargs[key]
                 kwargs = {**deployment, **nested_kwargs}
-                futures[deployment["model"]] = executor.submit(completion, **kwargs)
+                futures[deployment["model"]] = executor.submit(
+                    litellm.completion, **kwargs
+                )
 
             while futures:
                 # wait for the first returned future
@@ -246,7 +247,7 @@ def batch_completion_models_all_responses(*args, **kwargs):
 
     with concurrent.futures.ThreadPoolExecutor(max_workers=len(models)) as executor:
         for idx, model in enumerate(models):
-            future = executor.submit(completion, *args, model=model, **kwargs)
+            future = executor.submit(litellm.completion, *args, model=model, **kwargs)
             if future.result() is not None:
                 responses.append(future.result())
 

litellm/caching.py → litellm/caching/caching.py

@@ -212,7 +212,7 @@ def __init__(
 
         from litellm._service_logger import ServiceLogging
 
-        from ._redis import get_redis_client, get_redis_connection_pool
+        from .._redis import get_redis_client, get_redis_connection_pool
 
         redis_kwargs = {}
         if host is not None:
@@ -276,7 +276,7 @@ def __init__(
             )
 
     def init_async_client(self):
-        from ._redis import get_redis_async_client
+        from .._redis import get_redis_async_client
 
         return get_redis_async_client(
             connection_pool=self.async_redis_conn_pool, **self.redis_kwargs
@@ -302,7 +302,7 @@ def set_cache(self, key, value, **kwargs):
         except Exception as e:
             # NON blocking - notify users Redis is throwing an exception
             print_verbose(
-                f"LiteLLM Caching: set() - Got exception from REDIS : {str(e)}"
+                f"litellm.caching.caching: set() - Got exception from REDIS : {str(e)}"
             )
 
     def increment_cache(
@@ -705,7 +705,7 @@ def get_cache(self, key, **kwargs):
         except Exception as e:
             # NON blocking - notify users Redis is throwing an exception
             verbose_logger.error(
-                "LiteLLM Caching: get() - Got exception from REDIS: ", e
+                "litellm.caching.caching: get() - Got exception from REDIS: ", e
             )
 
     def batch_get_cache(self, key_list) -> dict:
@@ -781,7 +781,7 @@ async def async_get_cache(self, key, **kwargs):
                 )
                 # NON blocking - notify users Redis is throwing an exception
                 print_verbose(
-                    f"LiteLLM Caching: async get() - Got exception from REDIS: {str(e)}"
+                    f"litellm.caching.caching: async get() - Got exception from REDIS: {str(e)}"
                 )
 
     async def async_batch_get_cache(self, key_list) -> dict: