From 1fa42062f211117e39f191de195f7065def0996b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 15:47:36 -0700 Subject: [PATCH 01/20] fix(core_helpers.py): return None, instead of raising kwargs is None error Closes https://github.com/BerriAI/litellm/issues/6500 --- litellm/litellm_core_utils/core_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index f5619d237042..cddca61eec4a 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -80,7 +80,7 @@ def _get_parent_otel_span_from_kwargs( ) -> Union[Span, None]: try: if kwargs is None: - raise ValueError("kwargs is None") + return None litellm_params = kwargs.get("litellm_params") _metadata = kwargs.get("metadata") or {} if "litellm_parent_otel_span" in _metadata: From d866b12696ff614a56eb9dcddf383ba73ad3d161 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 15:49:40 -0700 Subject: [PATCH 02/20] docs(cost_tracking.md): cleanup doc --- docs/my-website/docs/proxy/cost_tracking.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/my-website/docs/proxy/cost_tracking.md b/docs/my-website/docs/proxy/cost_tracking.md index 033413099323..7f90273c39c3 100644 --- a/docs/my-website/docs/proxy/cost_tracking.md +++ b/docs/my-website/docs/proxy/cost_tracking.md @@ -284,9 +284,7 @@ Output from script :::info -Customer This is the value of `user_id` passed when calling [`/key/generate`](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post) - -[this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) +Customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm) - [LiteLLM API key](virtual_keys.md) From f147e33d593de506ba25e5290bca140eef4e57de Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 16:07:55 -0700 Subject: [PATCH 03/20] fix(vertex_and_google_ai_studio.py): handle function call with no params passed in Closes https://github.com/BerriAI/litellm/issues/6495 --- .../vertex_and_google_ai_studio_gemini.py | 8 ++- tests/llm_translation/test_optional_params.py | 18 ------ tests/llm_translation/test_vertex.py | 63 ++++++++++++++++++- tests/local_testing/test_function_calling.py | 31 +++++++++ 4 files changed, 99 insertions(+), 21 deletions(-) diff --git a/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py index a6e1d782a8d1..04a2d910a9bb 100644 --- a/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py @@ -419,9 +419,13 @@ def _map_function(self, value: List[dict]) -> List[Tools]: elif openai_function_object is not None: gtool_func_declaration = FunctionDeclaration( name=openai_function_object["name"], - description=openai_function_object.get("description", ""), - parameters=openai_function_object.get("parameters", {}), ) + _description = openai_function_object.get("description", None) + _parameters = openai_function_object.get("parameters", None) + if _description is not None: + gtool_func_declaration["description"] = _description + if _parameters is not None: + gtool_func_declaration["parameters"] = _parameters gtool_func_declarations.append(gtool_func_declaration) else: # assume it's a provider-specific param diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index f3cf8cb58e87..a0387ce1b2e1 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -784,21 +784,3 @@ def test_unmapped_vertex_anthropic_model(): max_retries=10, ) assert "max_retries" not in optional_params - - -@pytest.mark.parametrize( - "tools, key", - [ - ([{"googleSearchRetrieval": {}}], "googleSearchRetrieval"), - ([{"code_execution": {}}], "code_execution"), - ], -) -def test_vertex_tool_params(tools, key): - - optional_params = get_optional_params( - model="gemini-1.5-pro", - custom_llm_provider="vertex_ai", - tools=tools, - ) - print(optional_params) - assert optional_params["tools"][0][key] == {} diff --git a/tests/llm_translation/test_vertex.py b/tests/llm_translation/test_vertex.py index 4a9ef829db24..8bd1ddf3296a 100644 --- a/tests/llm_translation/test_vertex.py +++ b/tests/llm_translation/test_vertex.py @@ -12,8 +12,9 @@ sys.path.insert( 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - +import pytest import litellm +from litellm import get_optional_params def test_completion_pydantic_obj_2(): @@ -117,3 +118,63 @@ def test_build_vertex_schema(): assert new_schema["type"] == schema["type"] assert new_schema["properties"] == schema["properties"] assert "required" in new_schema and new_schema["required"] == schema["required"] + + +@pytest.mark.parametrize( + "tools, key", + [ + ([{"googleSearchRetrieval": {}}], "googleSearchRetrieval"), + ([{"code_execution": {}}], "code_execution"), + ], +) +def test_vertex_tool_params(tools, key): + + optional_params = get_optional_params( + model="gemini-1.5-pro", + custom_llm_provider="vertex_ai", + tools=tools, + ) + print(optional_params) + assert optional_params["tools"][0][key] == {} + + +@pytest.mark.parametrize( + "tool, expect_parameters", + [ + ( + { + "name": "test_function", + "description": "test_function_description", + "parameters": { + "type": "object", + "properties": {"test_param": {"type": "string"}}, + }, + }, + True, + ), + ( + { + "name": "test_function", + }, + False, + ), + ], +) +def test_vertex_function_translation(tool, expect_parameters): + """ + If param not set, don't set it in the request + """ + + tools = [tool] + optional_params = get_optional_params( + model="gemini-1.5-pro", + custom_llm_provider="vertex_ai", + tools=tools, + ) + print(optional_params) + if expect_parameters: + assert "parameters" in optional_params["tools"][0]["function_declarations"][0] + else: + assert ( + "parameters" not in optional_params["tools"][0]["function_declarations"][0] + ) diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py index 81d31186c042..851850a691f2 100644 --- a/tests/local_testing/test_function_calling.py +++ b/tests/local_testing/test_function_calling.py @@ -612,3 +612,34 @@ def test_passing_tool_result_as_list(): print(resp) assert resp.usage.prompt_tokens_details.cached_tokens > 0 + + +def test_function_calling_with_gemini(): + litellm.set_verbose = True + resp = litellm.completion( + model="gemini/gemini-1.5-pro-002", + messages=[ + { + "content": [ + { + "type": "text", + "text": "You are a helpful assistant that can interact with a computer to solve tasks.\n\n* If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.\n\n", + } + ], + "role": "system", + }, + { + "content": [{"type": "text", "text": "Hey, how's it going?"}], + "role": "user", + }, + ], + tools=[ + { + "type": "function", + "function": { + "name": "finish", + "description": "Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.", + }, + }, + ], + ) From dd309f86ae9867b063f8ae5a43a91ebbaec90132 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 16:21:42 -0700 Subject: [PATCH 04/20] test(test_router_timeout.py): add test for router timeout + retry logic --- tests/local_testing/test_router_timeout.py | 52 +++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/local_testing/test_router_timeout.py b/tests/local_testing/test_router_timeout.py index ccba7f676a71..a2dd03cc44ce 100644 --- a/tests/local_testing/test_router_timeout.py +++ b/tests/local_testing/test_router_timeout.py @@ -13,7 +13,7 @@ 0, os.path.abspath("../..") ) # Adds the parent directory to the system path - +from unittest.mock import patch, MagicMock, AsyncMock import os from dotenv import load_dotenv @@ -139,3 +139,53 @@ async def test_router_timeouts_bedrock(): pytest.fail( f"Did not raise error `openai.APITimeoutError`. Instead raised error type: {type(e)}, Error: {e}" ) + + +@pytest.mark.parametrize( + "num_retries, expected_call_count", + [(0, 1), (1, 2), (2, 3), (3, 4)], +) +def test_router_timeout_with_retries_anthropic_model(num_retries, expected_call_count): + """ + If request hits custom timeout, ensure it's retried. + """ + litellm._turn_on_debug() + from litellm.llms.custom_httpx.http_handler import HTTPHandler + import time + + # litellm.num_retries = num_retries + # litellm.request_timeout = 0.000001 + + router = Router( + model_list=[ + { + "model_name": "claude-3-haiku", + "litellm_params": { + "model": "anthropic/claude-3-haiku-20240307", + "timeout": 0.000001, + }, + } + ], + num_retries=num_retries, + ) + + custom_client = HTTPHandler() + + with patch.object(custom_client, "post", new=MagicMock()) as mock_client: + try: + + def delayed_response(*args, **kwargs): + time.sleep(0.01) # Exceeds the 0.000001 timeout + raise TimeoutError("Request timed out.") + + mock_client.side_effect = delayed_response + + router.completion( + model="claude-3-haiku", + messages=[{"role": "user", "content": "hello, who are u"}], + client=custom_client, + ) + except litellm.Timeout: + pass + + assert mock_client.call_count == expected_call_count From ba49202874f922b1979f0890fee10d71137034e7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 16:22:39 -0700 Subject: [PATCH 05/20] test: update test to use module level values --- tests/local_testing/test_router_timeout.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/local_testing/test_router_timeout.py b/tests/local_testing/test_router_timeout.py index a2dd03cc44ce..c13bc2deb1ba 100644 --- a/tests/local_testing/test_router_timeout.py +++ b/tests/local_testing/test_router_timeout.py @@ -153,8 +153,8 @@ def test_router_timeout_with_retries_anthropic_model(num_retries, expected_call_ from litellm.llms.custom_httpx.http_handler import HTTPHandler import time - # litellm.num_retries = num_retries - # litellm.request_timeout = 0.000001 + litellm.num_retries = num_retries + litellm.request_timeout = 0.000001 router = Router( model_list=[ @@ -162,11 +162,9 @@ def test_router_timeout_with_retries_anthropic_model(num_retries, expected_call_ "model_name": "claude-3-haiku", "litellm_params": { "model": "anthropic/claude-3-haiku-20240307", - "timeout": 0.000001, }, } ], - num_retries=num_retries, ) custom_client = HTTPHandler() From 149dd18bef4e8ded1f85d34ad7a2ac1066a415ef Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 12:17:35 +0530 Subject: [PATCH 06/20] (fix) Prometheus - Log Postgres DB latency, status on prometheus (#6484) * fix logging DB fails on prometheus * unit testing log to otel wrapper * unit testing for service logger + prometheus * use LATENCY buckets for service logging * fix service logging --- litellm/integrations/prometheus_services.py | 2 + litellm/proxy/proxy_config.yaml | 16 +-- litellm/proxy/utils.py | 61 ++++----- .../local_testing/test_prometheus_service.py | 71 ++++++++++ .../test_log_db_redis_services.py | 128 ++++++++++++++++++ 5 files changed, 229 insertions(+), 49 deletions(-) create mode 100644 tests/logging_callback_tests/test_log_db_redis_services.py diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index 53d20c067cd7..e657732dbbab 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -15,6 +15,7 @@ import litellm from litellm._logging import print_verbose, verbose_logger +from litellm.types.integrations.prometheus import LATENCY_BUCKETS from litellm.types.services import ServiceLoggerPayload, ServiceTypes @@ -96,6 +97,7 @@ def create_histogram(self, service: str, type_of_request: str): metric_name, "Latency for {} service".format(service), labelnames=[service], + buckets=LATENCY_BUCKETS, ) def create_counter(self, service: str, type_of_request: str): diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 18cc262b4543..5bc044526bf1 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,18 +3,8 @@ model_list: litellm_params: model: gpt-4o api_key: os.environ/OPENAI_API_KEY - tpm: 1000000 - rpm: 10000 - - -general_settings: - # master key is set via env var - # master_key: ####### - proxy_batch_write_at: 60 # Batch write spend updates every 60s + api_base: https://exampleopenaiendpoint-production.up.railway.app/ litellm_settings: - store_audit_logs: true - - # https://docs.litellm.ai/docs/proxy/reliability#default-fallbacks - default_fallbacks: ["gpt-4o-2024-08-06", "claude-3-5-sonnet-20240620"] - fallbacks: [{"gpt-4o-2024-08-06": ["claude-3-5-sonnet-20240620"]}, {"gpt-4o-2024-05-13": ["claude-3-5-sonnet-20240620"]}] \ No newline at end of file + callbacks: ["prometheus"] + service_callback: ["prometheus_system"] \ No newline at end of file diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4a10a0179193..9eab792ad5f8 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -140,25 +140,21 @@ def safe_deep_copy(data): def log_to_opentelemetry(func): @wraps(func) async def wrapper(*args, **kwargs): - start_time = datetime.now() + start_time: datetime = datetime.now() try: result = await func(*args, **kwargs) - end_time = datetime.now() + end_time: datetime = datetime.now() + + from litellm.proxy.proxy_server import proxy_logging_obj # Log to OTEL only if "parent_otel_span" is in kwargs and is not None - if ( - "parent_otel_span" in kwargs - and kwargs["parent_otel_span"] is not None - and "proxy_logging_obj" in kwargs - and kwargs["proxy_logging_obj"] is not None - ): - proxy_logging_obj = kwargs["proxy_logging_obj"] + if "PROXY" not in func.__name__: await proxy_logging_obj.service_logging_obj.async_service_success_hook( service=ServiceTypes.DB, call_type=func.__name__, - parent_otel_span=kwargs["parent_otel_span"], - duration=0.0, + parent_otel_span=kwargs.get("parent_otel_span", None), + duration=(end_time - start_time).total_seconds(), start_time=start_time, end_time=end_time, event_metadata={ @@ -179,8 +175,6 @@ async def wrapper(*args, **kwargs): kwargs=passed_kwargs ) if parent_otel_span is not None: - from litellm.proxy.proxy_server import proxy_logging_obj - metadata = get_litellm_metadata_from_kwargs(kwargs=passed_kwargs) await proxy_logging_obj.service_logging_obj.async_service_success_hook( service=ServiceTypes.BATCH_WRITE_TO_DB, @@ -194,28 +188,23 @@ async def wrapper(*args, **kwargs): # end of logging to otel return result except Exception as e: - end_time = datetime.now() - if ( - "parent_otel_span" in kwargs - and kwargs["parent_otel_span"] is not None - and "proxy_logging_obj" in kwargs - and kwargs["proxy_logging_obj"] is not None - ): - proxy_logging_obj = kwargs["proxy_logging_obj"] - await proxy_logging_obj.service_logging_obj.async_service_failure_hook( - error=e, - service=ServiceTypes.DB, - call_type=func.__name__, - parent_otel_span=kwargs["parent_otel_span"], - duration=0.0, - start_time=start_time, - end_time=end_time, - event_metadata={ - "function_name": func.__name__, - "function_kwargs": kwargs, - "function_args": args, - }, - ) + from litellm.proxy.proxy_server import proxy_logging_obj + + end_time: datetime = datetime.now() + await proxy_logging_obj.service_logging_obj.async_service_failure_hook( + error=e, + service=ServiceTypes.DB, + call_type=func.__name__, + parent_otel_span=kwargs.get("parent_otel_span"), + duration=(end_time - start_time).total_seconds(), + start_time=start_time, + end_time=end_time, + event_metadata={ + "function_name": func.__name__, + "function_kwargs": kwargs, + "function_args": args, + }, + ) raise e return wrapper @@ -348,6 +337,7 @@ def __init__( internal_usage_cache=self.internal_usage_cache.dual_cache, ) self.premium_user = premium_user + self.service_logging_obj = ServiceLogging() def startup_event( self, @@ -422,7 +412,6 @@ def update_values( self.internal_usage_cache.dual_cache.redis_cache = redis_cache def _init_litellm_callbacks(self, llm_router: Optional[litellm.Router] = None): - self.service_logging_obj = ServiceLogging() litellm.callbacks.append(self.max_parallel_request_limiter) # type: ignore litellm.callbacks.append(self.max_budget_limiter) # type: ignore litellm.callbacks.append(self.cache_control_check) # type: ignore diff --git a/tests/local_testing/test_prometheus_service.py b/tests/local_testing/test_prometheus_service.py index 86321ea2dfc1..49dd74839c29 100644 --- a/tests/local_testing/test_prometheus_service.py +++ b/tests/local_testing/test_prometheus_service.py @@ -11,6 +11,8 @@ from litellm import acompletion, Cache from litellm._service_logger import ServiceLogging from litellm.integrations.prometheus_services import PrometheusServicesLogger +from litellm.proxy.utils import ServiceTypes +from unittest.mock import patch, AsyncMock import litellm """ @@ -139,3 +141,72 @@ def get_azure_params(deployment_name: str): except Exception as e: pytest.fail(f"An exception occured - {str(e)}") + + +@pytest.mark.asyncio +async def test_service_logger_db_monitoring(): + """ + Test prometheus monitoring for database operations + """ + litellm.service_callback = ["prometheus_system"] + sl = ServiceLogging() + + # Create spy on prometheus logger's async_service_success_hook + with patch.object( + sl.prometheusServicesLogger, + "async_service_success_hook", + new_callable=AsyncMock, + ) as mock_prometheus_success: + # Test DB success monitoring + await sl.async_service_success_hook( + service=ServiceTypes.DB, + duration=0.3, + call_type="query", + event_metadata={"query_type": "SELECT", "table": "api_keys"}, + ) + + # Assert prometheus logger's success hook was called + mock_prometheus_success.assert_called_once() + # Optionally verify the payload + actual_payload = mock_prometheus_success.call_args[1]["payload"] + print("actual_payload sent to prometheus: ", actual_payload) + assert actual_payload.service == ServiceTypes.DB + assert actual_payload.duration == 0.3 + assert actual_payload.call_type == "query" + assert actual_payload.is_error is False + + +@pytest.mark.asyncio +async def test_service_logger_db_monitoring_failure(): + """ + Test prometheus monitoring for failed database operations + """ + litellm.service_callback = ["prometheus_system"] + sl = ServiceLogging() + + # Create spy on prometheus logger's async_service_failure_hook + with patch.object( + sl.prometheusServicesLogger, + "async_service_failure_hook", + new_callable=AsyncMock, + ) as mock_prometheus_failure: + # Test DB failure monitoring + test_error = Exception("Database connection failed") + await sl.async_service_failure_hook( + service=ServiceTypes.DB, + duration=0.3, + error=test_error, + call_type="query", + event_metadata={"query_type": "SELECT", "table": "api_keys"}, + ) + + # Assert prometheus logger's failure hook was called + mock_prometheus_failure.assert_called_once() + # Verify the payload + actual_payload = mock_prometheus_failure.call_args[1]["payload"] + print("actual_payload sent to prometheus: ", actual_payload) + assert actual_payload.service == ServiceTypes.DB + assert actual_payload.duration == 0.3 + assert actual_payload.call_type == "query" + assert actual_payload.is_error is True + assert actual_payload.error == "Database connection failed" diff --git a/tests/logging_callback_tests/test_log_db_redis_services.py b/tests/logging_callback_tests/test_log_db_redis_services.py new file mode 100644 index 000000000000..9f5db8009b3b --- /dev/null +++ b/tests/logging_callback_tests/test_log_db_redis_services.py @@ -0,0 +1,128 @@ +import io +import os +import sys + + +sys.path.insert(0, os.path.abspath("../..")) + +import asyncio +import gzip +import json +import logging +import time +from unittest.mock import AsyncMock, patch + +import pytest + +import litellm +from litellm import completion +from litellm._logging import verbose_logger +from litellm.proxy.utils import log_to_opentelemetry, ServiceTypes +from datetime import datetime + + +# Test async function to decorate +@log_to_opentelemetry +async def sample_db_function(*args, **kwargs): + return "success" + + +@log_to_opentelemetry +async def sample_proxy_function(*args, **kwargs): + return "success" + + +@pytest.mark.asyncio +async def test_log_to_opentelemetry_success(): + # Mock the proxy_logging_obj + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + # Setup mock + mock_proxy_logging.service_logging_obj.async_service_success_hook = AsyncMock() + + # Call the decorated function + result = await sample_db_function(parent_otel_span="test_span") + + # Assertions + assert result == "success" + mock_proxy_logging.service_logging_obj.async_service_success_hook.assert_called_once() + call_args = ( + mock_proxy_logging.service_logging_obj.async_service_success_hook.call_args[ + 1 + ] + ) + + assert call_args["service"] == ServiceTypes.DB + assert call_args["call_type"] == "sample_db_function" + assert call_args["parent_otel_span"] == "test_span" + assert isinstance(call_args["duration"], float) + assert isinstance(call_args["start_time"], datetime) + assert isinstance(call_args["end_time"], datetime) + assert "function_name" in call_args["event_metadata"] + + +@pytest.mark.asyncio +async def test_log_to_opentelemetry_duration(): + # Mock the proxy_logging_obj + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + # Setup mock + mock_proxy_logging.service_logging_obj.async_service_success_hook = AsyncMock() + + # Add a delay to the function to test duration + @log_to_opentelemetry + async def delayed_function(**kwargs): + await asyncio.sleep(1) # 1 second delay + return "success" + + # Call the decorated function + start = time.time() + result = await delayed_function(parent_otel_span="test_span") + end = time.time() + + # Get the actual duration + actual_duration = end - start + + # Get the logged duration from the mock call + call_args = ( + mock_proxy_logging.service_logging_obj.async_service_success_hook.call_args[ + 1 + ] + ) + logged_duration = call_args["duration"] + + # Assert the logged duration is approximately equal to actual duration (within 0.1 seconds) + assert abs(logged_duration - actual_duration) < 0.1 + assert result == "success" + + +@pytest.mark.asyncio +async def test_log_to_opentelemetry_failure(): + # Mock the proxy_logging_obj + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + # Setup mock + mock_proxy_logging.service_logging_obj.async_service_failure_hook = AsyncMock() + + # Create a failing function + @log_to_opentelemetry + async def failing_function(**kwargs): + raise ValueError("Test error") + + # Call the decorated function and expect it to raise + with pytest.raises(ValueError) as exc_info: + await failing_function(parent_otel_span="test_span") + + # Assertions + assert str(exc_info.value) == "Test error" + mock_proxy_logging.service_logging_obj.async_service_failure_hook.assert_called_once() + call_args = ( + mock_proxy_logging.service_logging_obj.async_service_failure_hook.call_args[ + 1 + ] + ) + + assert call_args["service"] == ServiceTypes.DB + assert call_args["call_type"] == "failing_function" + assert call_args["parent_otel_span"] == "test_span" + assert isinstance(call_args["duration"], float) + assert isinstance(call_args["start_time"], datetime) + assert isinstance(call_args["end_time"], datetime) + assert isinstance(call_args["error"], ValueError) From 0dc957c574cbdb9c4ea9180822cf96dd74af0bd8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 13:14:26 +0530 Subject: [PATCH 07/20] docs clarify vertex vs gemini --- docs/my-website/docs/providers/gemini.md | 20 ++++++++++++++++---- docs/my-website/docs/providers/vertex.md | 11 +++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 8a8d2a0046be..da83448c096f 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -4,11 +4,23 @@ import TabItem from '@theme/TabItem'; # Gemini - Google AI Studio -## Pre-requisites -* `pip install -q google-generativeai` -* Get API Key - https://aistudio.google.com/ +| Property | Details | +|-------|-------| +| Description | Google AI Studio is a fully-managed AI development platform for building and using generative AI. | +| Provider Route on LiteLLM | `gemini/` | +| Provider Doc | [Google AI Studio ↗](https://ai.google.dev/aistudio) | +| API Endpoint for Provider | https://generativelanguage.googleapis.com | + +
+ + +## API Keys + +```python +import os +os.environ["GEMINI_API_KEY"] = "your-api-key" +``` -# Gemini-Pro ## Sample Usage ```python from litellm import completion diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 852599cbec12..b69e8ee568c2 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -4,6 +4,17 @@ import TabItem from '@theme/TabItem'; # VertexAI [Anthropic, Gemini, Model Garden] + +| Property | Details | +|-------|-------| +| Description | Vertex AI is a fully-managed AI development platform for building and using generative AI. | +| Provider Route on LiteLLM | `vertex_ai/` | +| Link to Provider Doc | [Vertex AI ↗](https://cloud.google.com/vertex-ai) | +| Base URL | [https://{vertex_location}-aiplatform.googleapis.com/](https://{vertex_location}-aiplatform.googleapis.com/) | + +
+
+ Open In Colab From 37f5feb5f46d8ac44dd1cef3106bce7926269e59 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 21:07:17 +0530 Subject: [PATCH 08/20] (router_strategy/) ensure all async functions use async cache methods (#6489) * fix router strat * use async set / get cache in router_strategy * add coverage for router strategy * fix imports * fix batch_get_cache * use async methods for least busy * fix least busy use async methods * fix test_dual_cache_increment * test async_get_available_deployment when routing_strategy="least-busy" --- .circleci/config.yml | 1 + litellm/router.py | 11 ++ litellm/router_strategy/least_busy.py | 52 ++++++-- litellm/router_strategy/lowest_latency.py | 4 +- litellm/router_strategy/lowest_tpm_rpm.py | 12 +- .../test_router_strategy_async.py | 120 ++++++++++++++++++ tests/local_testing/test_dual_cache.py | 7 +- .../local_testing/test_least_busy_routing.py | 15 ++- 8 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 tests/code_coverage_tests/test_router_strategy_async.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8fcf51376c90..4734ee2a749f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -424,6 +424,7 @@ jobs: - run: ruff check ./litellm - run: python ./tests/documentation_tests/test_general_setting_keys.py - run: python ./tests/code_coverage_tests/router_code_coverage.py + - run: python ./tests/code_coverage_tests/test_router_strategy_async.py - run: python ./tests/documentation_tests/test_env_keys.py - run: helm lint ./deploy/charts/litellm-helm diff --git a/litellm/router.py b/litellm/router.py index 5ccdbcf4ae03..e2c033c6058c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5127,6 +5127,7 @@ async def async_get_available_deployment( and self.routing_strategy != "simple-shuffle" and self.routing_strategy != "cost-based-routing" and self.routing_strategy != "latency-based-routing" + and self.routing_strategy != "least-busy" ): # prevent regressions for other routing strategies, that don't have async get available deployments implemented. return self.get_available_deployment( model=model, @@ -5240,6 +5241,16 @@ async def async_get_available_deployment( healthy_deployments=healthy_deployments, model=model, ) + elif ( + self.routing_strategy == "least-busy" + and self.leastbusy_logger is not None + ): + deployment = ( + await self.leastbusy_logger.async_get_available_deployments( + model_group=model, + healthy_deployments=healthy_deployments, # type: ignore + ) + ) else: deployment = None if deployment is None: diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py index f1b35bb89d61..b1a85440f1bc 100644 --- a/litellm/router_strategy/least_busy.py +++ b/litellm/router_strategy/least_busy.py @@ -145,13 +145,14 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti request_count_api_key = f"{model_group}_request_count" # decrement count in cache request_count_dict = ( - self.router_cache.get_cache(key=request_count_api_key) or {} + await self.router_cache.async_get_cache(key=request_count_api_key) + or {} ) request_count_value: Optional[int] = request_count_dict.get(id, 0) if request_count_value is None: return request_count_dict[id] = request_count_value - 1 - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=request_count_api_key, value=request_count_dict ) @@ -178,13 +179,14 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti request_count_api_key = f"{model_group}_request_count" # decrement count in cache request_count_dict = ( - self.router_cache.get_cache(key=request_count_api_key) or {} + await self.router_cache.async_get_cache(key=request_count_api_key) + or {} ) request_count_value: Optional[int] = request_count_dict.get(id, 0) if request_count_value is None: return request_count_dict[id] = request_count_value - 1 - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=request_count_api_key, value=request_count_dict ) @@ -194,10 +196,14 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti except Exception: pass - def get_available_deployments(self, model_group: str, healthy_deployments: list): - request_count_api_key = f"{model_group}_request_count" - deployments = self.router_cache.get_cache(key=request_count_api_key) or {} - all_deployments = deployments + def _get_available_deployments( + self, + healthy_deployments: list, + all_deployments: dict, + ): + """ + Helper to get deployments using least busy strategy + """ for d in healthy_deployments: ## if healthy deployment not yet used if d["model_info"]["id"] not in all_deployments: @@ -219,3 +225,33 @@ def get_available_deployments(self, model_group: str, healthy_deployments: list) else: min_deployment = random.choice(healthy_deployments) return min_deployment + + def get_available_deployments( + self, + model_group: str, + healthy_deployments: list, + ): + """ + Sync helper to get deployments using least busy strategy + """ + request_count_api_key = f"{model_group}_request_count" + all_deployments = self.router_cache.get_cache(key=request_count_api_key) or {} + return self._get_available_deployments( + healthy_deployments=healthy_deployments, + all_deployments=all_deployments, + ) + + async def async_get_available_deployments( + self, model_group: str, healthy_deployments: list + ): + """ + Async helper to get deployments using least busy strategy + """ + request_count_api_key = f"{model_group}_request_count" + all_deployments = ( + await self.router_cache.async_get_cache(key=request_count_api_key) or {} + ) + return self._get_available_deployments( + healthy_deployments=healthy_deployments, + all_deployments=all_deployments, + ) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 287e6014628b..a96a8fa941f3 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -243,7 +243,7 @@ async def async_log_failure_event(self, kwargs, response_obj, start_time, end_ti "latency" ][: self.routing_args.max_latency_list_size - 1] + [1000.0] - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=latency_key, value=request_count_dict, ttl=self.routing_args.ttl, @@ -384,7 +384,7 @@ async def async_log_success_event( # noqa: PLR0915 request_count_dict[id][precise_minute].get("rpm", 0) + 1 ) - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=latency_key, value=request_count_dict, ttl=self.routing_args.ttl ) # reset map within window diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 45f32fbf041b..c79698ecf370 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -139,18 +139,22 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti # update cache ## TPM - request_count_dict = self.router_cache.get_cache(key=tpm_key) or {} + request_count_dict = ( + await self.router_cache.async_get_cache(key=tpm_key) or {} + ) request_count_dict[id] = request_count_dict.get(id, 0) + total_tokens - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=tpm_key, value=request_count_dict, ttl=self.routing_args.ttl ) ## RPM - request_count_dict = self.router_cache.get_cache(key=rpm_key) or {} + request_count_dict = ( + await self.router_cache.async_get_cache(key=rpm_key) or {} + ) request_count_dict[id] = request_count_dict.get(id, 0) + 1 - self.router_cache.set_cache( + await self.router_cache.async_set_cache( key=rpm_key, value=request_count_dict, ttl=self.routing_args.ttl ) diff --git a/tests/code_coverage_tests/test_router_strategy_async.py b/tests/code_coverage_tests/test_router_strategy_async.py new file mode 100644 index 000000000000..05bdca10f45c --- /dev/null +++ b/tests/code_coverage_tests/test_router_strategy_async.py @@ -0,0 +1,120 @@ +""" +Test that all cache calls in async functions in router_strategy/ are async + +""" + +import os +import sys +from typing import Dict, List, Tuple +import ast + +sys.path.insert( + 0, os.path.abspath("../..") +) # Adds the parent directory to the system path +import os + + +class AsyncCacheCallVisitor(ast.NodeVisitor): + def __init__(self): + self.async_functions: Dict[str, List[Tuple[str, int]]] = {} + self.current_function = None + + def visit_AsyncFunctionDef(self, node): + """Visit async function definitions and store their cache calls""" + self.current_function = node.name + self.async_functions[node.name] = [] + self.generic_visit(node) + self.current_function = None + + def visit_Call(self, node): + """Visit function calls and check for cache operations""" + if self.current_function is not None: + # Check if it's a cache-related call + if isinstance(node.func, ast.Attribute): + method_name = node.func.attr + if any(keyword in method_name.lower() for keyword in ["cache"]): + # Get the full method call path + if isinstance(node.func.value, ast.Name): + full_call = f"{node.func.value.id}.{method_name}" + elif isinstance(node.func.value, ast.Attribute): + # Handle nested attributes like self.router_cache.get + parts = [] + current = node.func.value + while isinstance(current, ast.Attribute): + parts.append(current.attr) + current = current.value + if isinstance(current, ast.Name): + parts.append(current.id) + parts.reverse() + parts.append(method_name) + full_call = ".".join(parts) + else: + full_call = method_name + # Store both the call and its line number + self.async_functions[self.current_function].append( + (full_call, node.lineno) + ) + self.generic_visit(node) + + +def get_python_files(directory: str) -> List[str]: + """Get all Python files in the router_strategy directory""" + python_files = [] + for file in os.listdir(directory): + if file.endswith(".py") and not file.startswith("__"): + python_files.append(os.path.join(directory, file)) + return python_files + + +def analyze_file(file_path: str) -> Dict[str, List[Tuple[str, int]]]: + """Analyze a Python file for async functions and their cache calls""" + with open(file_path, "r") as file: + tree = ast.parse(file.read()) + + visitor = AsyncCacheCallVisitor() + visitor.visit(tree) + return visitor.async_functions + + +def test_router_strategy_async_cache_calls(): + """Test that all cache calls in async functions are properly async""" + strategy_dir = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "litellm", + "router_strategy", + ) + + # Get all Python files in the router_strategy directory + python_files = get_python_files(strategy_dir) + + print("python files:", python_files) + + all_async_functions: Dict[str, Dict[str, List[Tuple[str, int]]]] = {} + + for file_path in python_files: + file_name = os.path.basename(file_path) + async_functions = analyze_file(file_path) + + if async_functions: + all_async_functions[file_name] = async_functions + print(f"\nAnalyzing {file_name}:") + + for func_name, cache_calls in async_functions.items(): + print(f"\nAsync function: {func_name}") + print(f"Cache calls found: {cache_calls}") + + # Assert that cache calls in async functions use async methods + for call, line_number in cache_calls: + if any(keyword in call.lower() for keyword in ["cache"]): + assert ( + "async" in call.lower() + ), f"VIOLATION: Cache call '{call}' in async function '{func_name}' should be async. file path: {file_path}, line number: {line_number}" + + # Assert we found async functions to analyze + assert ( + len(all_async_functions) > 0 + ), "No async functions found in router_strategy directory" + + +if __name__ == "__main__": + test_router_strategy_async_cache_calls() diff --git a/tests/local_testing/test_dual_cache.py b/tests/local_testing/test_dual_cache.py index d8c7cf3580a5..c3f3216d5d79 100644 --- a/tests/local_testing/test_dual_cache.py +++ b/tests/local_testing/test_dual_cache.py @@ -158,7 +158,7 @@ async def test_dual_cache_batch_operations(is_async): if is_async: results = await dual_cache.async_batch_get_cache(test_keys) else: - results = dual_cache.batch_get_cache(test_keys) + results = dual_cache.batch_get_cache(test_keys, parent_otel_span=None) assert results == test_values mock_redis_get.assert_not_called() @@ -181,7 +181,10 @@ async def test_dual_cache_increment(is_async): ) as mock_redis_increment: if is_async: result = await dual_cache.async_increment_cache( - test_key, increment_value, local_only=True + test_key, + increment_value, + local_only=True, + parent_otel_span=None, ) else: result = dual_cache.increment_cache( diff --git a/tests/local_testing/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py index dc7db95602ba..c9c6eb609365 100644 --- a/tests/local_testing/test_least_busy_routing.py +++ b/tests/local_testing/test_least_busy_routing.py @@ -65,7 +65,9 @@ def test_get_available_deployments(): # test_get_available_deployments() -def test_router_get_available_deployments(): +@pytest.mark.parametrize("async_test", [True, False]) +@pytest.mark.asyncio +async def test_router_get_available_deployments(async_test): """ Tests if 'get_available_deployments' returns the least busy deployment """ @@ -114,9 +116,14 @@ def test_router_get_available_deployments(): deployment = "azure/chatgpt-v-2" request_count_dict = {1: 10, 2: 54, 3: 100} cache_key = f"{model_group}_request_count" - router.cache.set_cache(key=cache_key, value=request_count_dict) - - deployment = router.get_available_deployment(model=model_group, messages=None) + if async_test is True: + await router.cache.async_set_cache(key=cache_key, value=request_count_dict) + deployment = await router.async_get_available_deployment( + model=model_group, messages=None + ) + else: + router.cache.set_cache(key=cache_key, value=request_count_dict) + deployment = router.get_available_deployment(model=model_group, messages=None) print(f"deployment: {deployment}") assert deployment["model_info"]["id"] == "1" From 947af8c66079eb768edff1fe43f525b9a08cc327 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 21:28:14 +0530 Subject: [PATCH 09/20] (fix) proxy - fix when `STORE_MODEL_IN_DB` should be set (#6492) * set store_model_in_db at the top * correctly use store_model_in_db global --- litellm/proxy/proxy_server.py | 15 ++++++++++----- litellm/secret_managers/main.py | 23 +++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cadb6063c786..1f0271f34517 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -250,7 +250,12 @@ def generate_feedback_box(): load_aws_secret_manager, ) from litellm.secret_managers.google_kms import load_google_kms -from litellm.secret_managers.main import get_secret, get_secret_str, str_to_bool +from litellm.secret_managers.main import ( + get_secret, + get_secret_bool, + get_secret_str, + str_to_bool, +) from litellm.types.integrations.slack_alerting import SlackAlertingArgs from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, @@ -2894,9 +2899,9 @@ async def initialize_scheduled_background_jobs( proxy_budget_rescheduler_max_time: int, proxy_batch_write_at: int, proxy_logging_obj: ProxyLogging, - store_model_in_db: bool, ): """Initializes scheduled background jobs""" + global store_model_in_db scheduler = AsyncIOScheduler() interval = random.randint( proxy_budget_rescheduler_min_time, proxy_budget_rescheduler_max_time @@ -2921,8 +2926,9 @@ async def initialize_scheduled_background_jobs( ### ADD NEW MODELS ### store_model_in_db = ( - get_secret("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db - ) # type: ignore + get_secret_bool("STORE_MODEL_IN_DB", store_model_in_db) or store_model_in_db + ) + if store_model_in_db is True: scheduler.add_job( proxy_config.add_deployment, @@ -3141,7 +3147,6 @@ async def startup_event(): proxy_budget_rescheduler_max_time=proxy_budget_rescheduler_max_time, proxy_batch_write_at=proxy_batch_write_at, proxy_logging_obj=proxy_logging_obj, - store_model_in_db=store_model_in_db, ) diff --git a/litellm/secret_managers/main.py b/litellm/secret_managers/main.py index 522f2bc39e61..f3d6d420ac76 100644 --- a/litellm/secret_managers/main.py +++ b/litellm/secret_managers/main.py @@ -67,6 +67,29 @@ def get_secret_str( return value +def get_secret_bool( + secret_name: str, + default_value: Optional[bool] = None, +) -> Optional[bool]: + """ + Guarantees response from 'get_secret' is either boolean or none. Used for fixing linting errors. + + Args: + secret_name: The name of the secret to get. + default_value: The default value to return if the secret is not found. + + Returns: + The secret value as a boolean or None if the secret is not found. + """ + _secret_value = get_secret(secret_name, default_value) + if _secret_value is None: + return None + elif isinstance(_secret_value, bool): + return _secret_value + else: + return str_to_bool(_secret_value) + + def get_secret( # noqa: PLR0915 secret_name: str, default_value: Optional[Union[str, bool]] = None, From 17695e1c8f432530b6270731df3adae3e213998a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 21:29:19 +0530 Subject: [PATCH 10/20] (fix) `PrometheusServicesLogger` `_get_metric` should return metric in Registry (#6486) * fix logging DB fails on prometheus * unit testing log to otel wrapper * unit testing for service logger + prometheus * use LATENCY buckets for service logging * fix service logging * fix _get_metric in prom services logger * add clear doc string * unit testing for prom service logger --- litellm/integrations/prometheus_services.py | 15 ++-- litellm/proxy/proxy_config.yaml | 3 +- litellm/proxy/utils.py | 8 +- .../local_testing/test_prometheus_service.py | 85 +++++++++++++++++++ 4 files changed, 100 insertions(+), 11 deletions(-) diff --git a/litellm/integrations/prometheus_services.py b/litellm/integrations/prometheus_services.py index e657732dbbab..a36ac9b9c024 100644 --- a/litellm/integrations/prometheus_services.py +++ b/litellm/integrations/prometheus_services.py @@ -81,18 +81,17 @@ def is_metric_registered(self, metric_name) -> bool: return True return False - def get_metric(self, metric_name): - for metric in self.REGISTRY.collect(): - for sample in metric.samples: - if metric_name == sample.name: - return metric - return None + def _get_metric(self, metric_name): + """ + Helper function to get a metric from the registry by name. + """ + return self.REGISTRY._names_to_collectors.get(metric_name) def create_histogram(self, service: str, type_of_request: str): metric_name = "litellm_{}_{}".format(service, type_of_request) is_registered = self.is_metric_registered(metric_name) if is_registered: - return self.get_metric(metric_name) + return self._get_metric(metric_name) return self.Histogram( metric_name, "Latency for {} service".format(service), @@ -104,7 +103,7 @@ def create_counter(self, service: str, type_of_request: str): metric_name = "litellm_{}_{}".format(service, type_of_request) is_registered = self.is_metric_registered(metric_name) if is_registered: - return self.get_metric(metric_name) + return self._get_metric(metric_name) return self.Counter( metric_name, "Total {} for {} service".format(type_of_request, service), diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 5bc044526bf1..95eff095c22e 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -7,4 +7,5 @@ model_list: litellm_settings: callbacks: ["prometheus"] - service_callback: ["prometheus_system"] \ No newline at end of file + service_callback: ["prometheus_system"] + cache: true diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9eab792ad5f8..656e97a6c5ce 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -138,6 +138,12 @@ def safe_deep_copy(data): def log_to_opentelemetry(func): + """ + Decorator to log the duration of a DB related function to ServiceLogger() + + Handles logging DB success/failure to ServiceLogger(), which logs to Prometheus, OTEL, Datadog + """ + @wraps(func) async def wrapper(*args, **kwargs): start_time: datetime = datetime.now() @@ -145,10 +151,8 @@ async def wrapper(*args, **kwargs): try: result = await func(*args, **kwargs) end_time: datetime = datetime.now() - from litellm.proxy.proxy_server import proxy_logging_obj - # Log to OTEL only if "parent_otel_span" is in kwargs and is not None if "PROXY" not in func.__name__: await proxy_logging_obj.service_logging_obj.async_service_success_hook( service=ServiceTypes.DB, diff --git a/tests/local_testing/test_prometheus_service.py b/tests/local_testing/test_prometheus_service.py index 49dd74839c29..c640532a07e7 100644 --- a/tests/local_testing/test_prometheus_service.py +++ b/tests/local_testing/test_prometheus_service.py @@ -210,3 +210,88 @@ async def test_service_logger_db_monitoring_failure(): assert actual_payload.call_type == "query" assert actual_payload.is_error is True assert actual_payload.error == "Database connection failed" + + +def test_get_metric_existing(): + """Test _get_metric when metric exists. _get_metric should return the metric object""" + pl = PrometheusServicesLogger() + # Create a metric first + hist = pl.create_histogram( + service="test_service", type_of_request="test_type_of_request" + ) + + # Test retrieving existing metric + retrieved_metric = pl._get_metric("litellm_test_service_test_type_of_request") + assert retrieved_metric is hist + assert retrieved_metric is not None + + +def test_get_metric_non_existing(): + """Test _get_metric when metric doesn't exist, returns None""" + pl = PrometheusServicesLogger() + + # Test retrieving non-existent metric + non_existent = pl._get_metric("non_existent_metric") + assert non_existent is None + + +def test_create_histogram_new(): + """Test creating a new histogram""" + pl = PrometheusServicesLogger() + + # Create new histogram + hist = pl.create_histogram( + service="test_service", type_of_request="test_type_of_request" + ) + + assert hist is not None + assert pl._get_metric("litellm_test_service_test_type_of_request") is hist + + +def test_create_histogram_existing(): + """Test creating a histogram that already exists""" + pl = PrometheusServicesLogger() + + # Create initial histogram + hist1 = pl.create_histogram( + service="test_service", type_of_request="test_type_of_request" + ) + + # Create same histogram again + hist2 = pl.create_histogram( + service="test_service", type_of_request="test_type_of_request" + ) + + assert hist2 is hist1 # same object + assert pl._get_metric("litellm_test_service_test_type_of_request") is hist1 + + +def test_create_counter_new(): + """Test creating a new counter""" + pl = PrometheusServicesLogger() + + # Create new counter + counter = pl.create_counter( + service="test_service", type_of_request="test_type_of_request" + ) + + assert counter is not None + assert pl._get_metric("litellm_test_service_test_type_of_request") is counter + + +def test_create_counter_existing(): + """Test creating a counter that already exists""" + pl = PrometheusServicesLogger() + + # Create initial counter + counter1 = pl.create_counter( + service="test_service", type_of_request="test_type_of_request" + ) + + # Create same counter again + counter2 = pl.create_counter( + service="test_service", type_of_request="test_type_of_request" + ) + + assert counter2 is counter1 + assert pl._get_metric("litellm_test_service_test_type_of_request") is counter1 From fe811a50da5c4339822829de79652b969a8d9d1a Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 29 Oct 2024 21:29:44 +0530 Subject: [PATCH 11/20] =?UTF-8?q?bump:=20version=201.51.0=20=E2=86=92=201.?= =?UTF-8?q?51.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a9dde2bfcdca..b4768e3024d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.51.0" +version = "1.51.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -91,7 +91,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.51.0" +version = "1.51.1" version_files = [ "pyproject.toml:^version" ] From 74d7216faf5aa5dafba53f451eca0e2176f53462 Mon Sep 17 00:00:00 2001 From: Xingyao Wang Date: Tue, 29 Oct 2024 11:02:42 -0500 Subject: [PATCH 12/20] Add `azure/gpt-4o-mini-2024-07-18` to model_prices_and_context_window.json (#6477) --- model_prices_and_context_window.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8f833c129cae..9578ed9eaeac 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -774,6 +774,20 @@ "supports_vision": true, "supports_prompt_caching": true }, + "azure/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000075, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, From 3b385fbedc8971324cb759f2d41049f5e5f92faf Mon Sep 17 00:00:00 2001 From: vibhanshu-ob <115142120+vibhanshu-ob@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:36:23 +0530 Subject: [PATCH 13/20] Update utils.py (#6468) Fixed missing keys --- litellm/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 5f86fd894cb4..cdf77f1a5153 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -5361,7 +5361,7 @@ def validate_environment( # noqa: PLR0915 if "CODESTRAL_API_KEY" in os.environ: keys_in_environment = True else: - missing_keys.append("GROQ_API_KEY") + missing_keys.append("CODESTRAL_API_KEY") elif custom_llm_provider == "deepseek": if "DEEPSEEK_API_KEY" in os.environ: keys_in_environment = True @@ -5451,7 +5451,7 @@ def validate_environment( # noqa: PLR0915 if "VERTEXAI_PROJECT" in os.environ and "VERTEXAI_LOCATION" in os.environ: keys_in_environment = True else: - missing_keys.extend(["VERTEXAI_PROJECT", "VERTEXAI_PROJECT"]) + missing_keys.extend(["VERTEXAI_PROJECT", "VERTEXAI_LOCATION"]) ## huggingface elif model in litellm.huggingface_models: if "HUGGINGFACE_API_KEY" in os.environ: From 6b991e74afa25a1e872c514aa81dbd63a3c55eb7 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 29 Oct 2024 13:58:29 -0700 Subject: [PATCH 14/20] (perf) Litellm redis router fix - ~100ms improvement (#6483) * docs(exception_mapping.md): add missing exception types Fixes https://github.com/Aider-AI/aider/issues/2120#issuecomment-2438971183 * fix(main.py): register custom model pricing with specific key Ensure custom model pricing is registered to the specific model+provider key combination * test: make testing more robust for custom pricing * fix(redis_cache.py): instrument otel logging for sync redis calls ensures complete coverage for all redis cache calls * refactor: pass parent_otel_span for redis caching calls in router allows for more observability into what calls are causing latency issues * test: update tests with new params * refactor: ensure e2e otel tracing for router * refactor(router.py): add more otel tracing acrosss router catch all latency issues for router requests * fix: fix linting error * fix(router.py): fix linting error * fix: fix test * test: fix tests * fix(dual_cache.py): pass ttl to redis cache * fix: fix param * perf(cooldown_cache.py): improve cooldown cache, to store cache results in memory for 5s, prevents redis call from being made on each request reduces 100ms latency per call with caching enabled on router * fix: fix test * fix(cooldown_cache.py): handle if a result is None * fix(cooldown_cache.py): add debug statements * refactor(dual_cache.py): move to using an in-memory check for batch get cache, to prevent redis from being hit for every call * fix(cooldown_cache.py): fix linting erropr --- litellm/caching/caching.py | 5 +- litellm/caching/dual_cache.py | 107 +++++++++++++----- litellm/router.py | 3 +- litellm/router_utils/cooldown_cache.py | 36 ++++-- tests/local_testing/test_acooldowns_router.py | 2 + tests/local_testing/test_caching.py | 28 +++++ tests/local_testing/test_router.py | 2 + tests/local_testing/test_unit_test_caching.py | 2 +- .../test_router_cooldown_utils.py | 2 +- 9 files changed, 143 insertions(+), 44 deletions(-) diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index f845633cb90b..5fd972a76ffd 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -271,7 +271,7 @@ def get_cache_key(self, *args, **kwargs) -> str: cache_key += f"{str(param)}: {str(param_value)}" verbose_logger.debug("\nCreated cache key: %s", cache_key) - hashed_cache_key = self._get_hashed_cache_key(cache_key) + hashed_cache_key = Cache._get_hashed_cache_key(cache_key) hashed_cache_key = self._add_redis_namespace_to_cache_key( hashed_cache_key, **kwargs ) @@ -431,7 +431,8 @@ def _get_kwargs_to_exclude_from_cache_key(self) -> Set[str]: """ return set(["metadata"]) - def _get_hashed_cache_key(self, cache_key: str) -> str: + @staticmethod + def _get_hashed_cache_key(cache_key: str) -> str: """ Get the hashed cache key for the given cache key. diff --git a/litellm/caching/dual_cache.py b/litellm/caching/dual_cache.py index 35659b865b59..1bf16bb65465 100644 --- a/litellm/caching/dual_cache.py +++ b/litellm/caching/dual_cache.py @@ -8,8 +8,9 @@ - async_get_cache """ +import time import traceback -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional, Tuple import litellm from litellm._logging import print_verbose, verbose_logger @@ -25,6 +26,19 @@ else: Span = Any +from collections import OrderedDict + + +class LimitedSizeOrderedDict(OrderedDict): + def __init__(self, *args, max_size=100, **kwargs): + super().__init__(*args, **kwargs) + self.max_size = max_size + + def __setitem__(self, key, value): + # If inserting a new key exceeds max size, remove the oldest item + if len(self) >= self.max_size: + self.popitem(last=False) + super().__setitem__(key, value) class DualCache(BaseCache): """ @@ -39,13 +53,18 @@ def __init__( redis_cache: Optional[RedisCache] = None, default_in_memory_ttl: Optional[float] = None, default_redis_ttl: Optional[float] = None, + default_redis_batch_cache_expiry: float = 1, + default_max_redis_batch_cache_size: int = 100, ) -> None: super().__init__() # If in_memory_cache is not provided, use the default InMemoryCache self.in_memory_cache = in_memory_cache or InMemoryCache() # If redis_cache is not provided, use the default RedisCache self.redis_cache = redis_cache - + self.last_redis_batch_access_time = LimitedSizeOrderedDict( + max_size=default_max_redis_batch_cache_size + ) + self.redis_batch_cache_expiry = default_redis_batch_cache_expiry self.default_in_memory_ttl = ( default_in_memory_ttl or litellm.default_in_memory_ttl ) @@ -150,20 +169,34 @@ def batch_get_cache( - for the none values in the result - check the redis cache """ - sublist_keys = [ - key for key, value in zip(keys, result) if value is None - ] - # If not found in in-memory cache, try fetching from Redis - redis_result = self.redis_cache.batch_get_cache( - sublist_keys, parent_otel_span=parent_otel_span - ) - if redis_result is not None: - # Update in-memory cache with the value from Redis - for key in redis_result: - self.in_memory_cache.set_cache(key, redis_result[key], **kwargs) + # Track the last access time for these keys + current_time = time.time() + key_tuple = tuple(keys) + + # Only hit Redis if the last access time was more than 5 seconds ago + if ( + key_tuple not in self.last_redis_batch_access_time + or current_time - self.last_redis_batch_access_time[key_tuple] + >= self.redis_batch_cache_expiry + ): + + sublist_keys = [ + key for key, value in zip(keys, result) if value is None + ] + # If not found in in-memory cache, try fetching from Redis + redis_result = self.redis_cache.batch_get_cache( + sublist_keys, parent_otel_span=parent_otel_span + ) + if redis_result is not None: + # Update in-memory cache with the value from Redis + for key in redis_result: + self.in_memory_cache.set_cache( + key, redis_result[key], **kwargs + ) - for key, value in redis_result.items(): - result[keys.index(key)] = value + + for key, value in redis_result.items(): + result[keys.index(key)] = value print_verbose(f"async batch get cache: cache result: {result}") return result @@ -227,29 +260,41 @@ async def async_batch_get_cache( if in_memory_result is not None: result = in_memory_result + if None in result and self.redis_cache is not None and local_only is False: """ - for the none values in the result - check the redis cache """ - sublist_keys = [ - key for key, value in zip(keys, result) if value is None - ] - # If not found in in-memory cache, try fetching from Redis - redis_result = await self.redis_cache.async_batch_get_cache( - sublist_keys, parent_otel_span=parent_otel_span - ) + # Track the last access time for these keys + current_time = time.time() + key_tuple = tuple(keys) + + # Only hit Redis if the last access time was more than 5 seconds ago + if ( + key_tuple not in self.last_redis_batch_access_time + or current_time - self.last_redis_batch_access_time[key_tuple] + >= self.redis_batch_cache_expiry + ): + sublist_keys = [ + key for key, value in zip(keys, result) if value is None + ] + # If not found in in-memory cache, try fetching from Redis + redis_result = await self.redis_cache.async_batch_get_cache( + sublist_keys, parent_otel_span=parent_otel_span + ) - if redis_result is not None: - # Update in-memory cache with the value from Redis + + if redis_result is not None: + # Update in-memory cache with the value from Redis + for key, value in redis_result.items(): + if value is not None: + await self.in_memory_cache.async_set_cache( + key, redis_result[key], **kwargs + ) for key, value in redis_result.items(): - if value is not None: - await self.in_memory_cache.async_set_cache( - key, redis_result[key], **kwargs - ) - for key, value in redis_result.items(): - index = keys.index(key) - result[index] = value + index = keys.index(key) + result[index] = value return result except Exception: diff --git a/litellm/router.py b/litellm/router.py index e2c033c6058c..ac26aa61e985 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5153,6 +5153,7 @@ async def async_get_available_deployment( verbose_router_logger.debug( f"async cooldown deployments: {cooldown_deployments}" ) + verbose_router_logger.debug(f"cooldown_deployments: {cooldown_deployments}") healthy_deployments = self._filter_cooldown_deployments( healthy_deployments=healthy_deployments, cooldown_deployments=cooldown_deployments, @@ -5261,7 +5262,7 @@ async def async_get_available_deployment( _cooldown_time = self.cooldown_cache.get_min_cooldown( model_ids=model_ids, parent_otel_span=parent_otel_span ) - _cooldown_list = _get_cooldown_deployments( + _cooldown_list = await _async_get_cooldown_deployments( litellm_router_instance=self, parent_otel_span=parent_otel_span ) raise RouterRateLimitError( diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index 792d91811a71..44174f3b16c6 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -7,7 +7,15 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict from litellm import verbose_logger -from litellm.caching.caching import DualCache +from litellm.caching.caching import Cache, DualCache +from litellm.caching.in_memory_cache import InMemoryCache + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span = _Span +else: + Span = Any if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -28,6 +36,7 @@ class CooldownCache: def __init__(self, cache: DualCache, default_cooldown_time: float): self.cache = cache self.default_cooldown_time = default_cooldown_time + self.in_memory_cache = InMemoryCache() def _common_add_cooldown_logic( self, model_id: str, original_exception, exception_status, cooldown_time: float @@ -83,21 +92,32 @@ def add_deployment_to_cooldown( ) raise e + @staticmethod + def get_cooldown_cache_key(model_id: str) -> str: + return f"deployment:{model_id}:cooldown" + async def async_get_active_cooldowns( self, model_ids: List[str], parent_otel_span: Optional[Span] ) -> List[Tuple[str, CooldownCacheValue]]: # Generate the keys for the deployments - keys = [f"deployment:{model_id}:cooldown" for model_id in model_ids] + keys = [ + CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids + ] # Retrieve the values for the keys using mget - results = ( - await self.cache.async_batch_get_cache( - keys=keys, parent_otel_span=parent_otel_span - ) - or [] + ## more likely to be none if no models ratelimited. So just check redis every 1s + ## each redis call adds ~100ms latency. + + ## check in memory cache first + results = await self.cache.async_batch_get_cache( + keys=keys, parent_otel_span=parent_otel_span ) + active_cooldowns: List[Tuple[str, CooldownCacheValue]] = [] + + if results is None: + return active_cooldowns + - active_cooldowns = [] # Process the results for model_id, result in zip(model_ids, results): if result and isinstance(result, dict): diff --git a/tests/local_testing/test_acooldowns_router.py b/tests/local_testing/test_acooldowns_router.py index cad4d9e66972..f186d42f1855 100644 --- a/tests/local_testing/test_acooldowns_router.py +++ b/tests/local_testing/test_acooldowns_router.py @@ -17,6 +17,7 @@ from dotenv import load_dotenv import litellm + from litellm import Router load_dotenv() @@ -130,6 +131,7 @@ def test_multiple_deployments_parallel(): @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_cooldown_same_model_name(sync_mode): + litellm._turn_on_debug() # users could have the same model with different api_base # example # azure/chatgpt, api_base: 1234 diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index b195854309fe..3456f4535dd1 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -681,6 +681,7 @@ async def test_redis_cache_basic(): @pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) async def test_redis_batch_cache_write(): """ Init redis client @@ -2477,3 +2478,30 @@ async def test_redis_caching_ttl_sadd(): ) print(f"expected_timedelta: {expected_timedelta}") assert mock_expire.call_args.args[1] == expected_timedelta + + +@pytest.mark.asyncio() +async def test_dual_cache_caching_batch_get_cache(): + """ + - check redis cache called for initial batch get cache + - check redis cache not called for consecutive batch get cache with same keys + """ + from litellm.caching.dual_cache import DualCache + from litellm.caching.redis_cache import RedisCache + + dc = DualCache(redis_cache=MagicMock(spec=RedisCache)) + + with patch.object( + dc.redis_cache, + "async_batch_get_cache", + new=AsyncMock( + return_value={"test_key1": "test_value1", "test_key2": "test_value2"} + ), + ) as mock_async_get_cache: + await dc.async_batch_get_cache(keys=["test_key1", "test_key2"]) + + assert mock_async_get_cache.call_count == 1 + + await dc.async_batch_get_cache(keys=["test_key1", "test_key2"]) + + assert mock_async_get_cache.call_count == 1 diff --git a/tests/local_testing/test_router.py b/tests/local_testing/test_router.py index d360d73176aa..7bf0b0bba0cf 100644 --- a/tests/local_testing/test_router.py +++ b/tests/local_testing/test_router.py @@ -2445,6 +2445,8 @@ def _return_exception(*args, **kwargs): except litellm.RateLimitError: pass + await asyncio.sleep(2) + if sync_mode: cooldown_deployments = _get_cooldown_deployments( litellm_router_instance=router, parent_otel_span=None diff --git a/tests/local_testing/test_unit_test_caching.py b/tests/local_testing/test_unit_test_caching.py index 4d7c506666bb..5f8f41ba54e8 100644 --- a/tests/local_testing/test_unit_test_caching.py +++ b/tests/local_testing/test_unit_test_caching.py @@ -135,7 +135,7 @@ def test_get_cache_key_text_completion(): def test_get_hashed_cache_key(): cache = Cache() cache_key = "model:gpt-3.5-turbo,messages:Hello world" - hashed_key = cache._get_hashed_cache_key(cache_key) + hashed_key = Cache._get_hashed_cache_key(cache_key) assert len(hashed_key) == 64 # SHA-256 produces a 64-character hex string diff --git a/tests/router_unit_tests/test_router_cooldown_utils.py b/tests/router_unit_tests/test_router_cooldown_utils.py index c8795e541a52..7ee2e927dc6e 100644 --- a/tests/router_unit_tests/test_router_cooldown_utils.py +++ b/tests/router_unit_tests/test_router_cooldown_utils.py @@ -11,7 +11,7 @@ from concurrent.futures import ThreadPoolExecutor from collections import defaultdict from dotenv import load_dotenv -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from litellm.integrations.prometheus import PrometheusLogger from litellm.router_utils.cooldown_callbacks import router_cooldown_event_callback from litellm.router_utils.cooldown_handlers import ( From 9d2555262dd760ebefefee1339738133dc271105 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 17:08:45 -0700 Subject: [PATCH 15/20] refactor(prometheus.py): move to using standard logging payload for reading the remaining request / tokens Ensures prometheus token tracking works for anthropic as well --- litellm/integrations/prometheus.py | 38 +++++++---- litellm/litellm_core_utils/litellm_logging.py | 67 ++++++++++++++----- ...odel_prices_and_context_window_backup.json | 14 ++++ litellm/proxy/_new_secret_config.yaml | 8 +-- litellm/types/utils.py | 9 ++- .../test_standard_logging_payload.py | 39 +++++++++++ 6 files changed, 142 insertions(+), 33 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 01c08ef04ec0..1ec2cc71cfa5 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -849,9 +849,13 @@ def set_llm_deployment_success_metrics( ): try: verbose_logger.debug("setting remaining tokens requests metric") - standard_logging_payload: StandardLoggingPayload = request_kwargs.get( - "standard_logging_object", {} + standard_logging_payload: Optional[StandardLoggingPayload] = ( + request_kwargs.get("standard_logging_object") ) + + if standard_logging_payload is None: + return + model_group = standard_logging_payload["model_group"] api_base = standard_logging_payload["api_base"] _response_headers = request_kwargs.get("response_headers") @@ -862,22 +866,30 @@ def set_llm_deployment_success_metrics( _model_info = _metadata.get("model_info") or {} model_id = _model_info.get("id", None) - remaining_requests = None - remaining_tokens = None + remaining_requests: Optional[int] = None + remaining_tokens: Optional[int] = None + if additional_headers := standard_logging_payload["hidden_params"][ + "additional_headers" + ]: + remaining_requests = additional_headers.get( + "x_ratelimit_remaining_requests", None + ) + remaining_tokens = additional_headers.get( + "x_ratelimit_remaining_tokens", None + ) # OpenAI / OpenAI Compatible headers if ( - _response_headers - and "x-ratelimit-remaining-requests" in _response_headers + additional_headers + and "x_ratelimit_remaining_requests" in additional_headers ): - remaining_requests = _response_headers["x-ratelimit-remaining-requests"] + remaining_requests = additional_headers[ + "x_ratelimit_remaining_requests" + ] if ( - _response_headers - and "x-ratelimit-remaining-tokens" in _response_headers + additional_headers + and "x_ratelimit_remaining_tokens" in additional_headers ): - remaining_tokens = _response_headers["x-ratelimit-remaining-tokens"] - verbose_logger.debug( - f"remaining requests: {remaining_requests}, remaining tokens: {remaining_tokens}" - ) + remaining_tokens = additional_headers["x_ratelimit_remaining_tokens"] if remaining_requests: """ diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 7f403e422c54..a35fbace7af6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -42,6 +42,7 @@ ImageResponse, ModelResponse, StandardCallbackDynamicParams, + StandardLoggingAdditionalHeaders, StandardLoggingHiddenParams, StandardLoggingMetadata, StandardLoggingModelCostFailureDebugInformation, @@ -2640,6 +2641,52 @@ def get_final_response_obj( return final_response_obj + @staticmethod + def get_additional_headers( + additiona_headers: Optional[dict], + ) -> Optional[StandardLoggingAdditionalHeaders]: + + if additiona_headers is None: + return None + + additional_logging_headers: StandardLoggingAdditionalHeaders = {} + + for key in StandardLoggingAdditionalHeaders.__annotations__.keys(): + _key = key.lower() + _key = _key.replace("_", "-") + if _key in additiona_headers: + try: + additional_logging_headers[key] = int(additiona_headers[_key]) # type: ignore + except (ValueError, TypeError): + verbose_logger.debug( + f"Could not convert {additiona_headers[_key]} to int for key {key}." + ) + return additional_logging_headers + + @staticmethod + def get_hidden_params( + hidden_params: Optional[dict], + ) -> StandardLoggingHiddenParams: + clean_hidden_params = StandardLoggingHiddenParams( + model_id=None, + cache_key=None, + api_base=None, + response_cost=None, + additional_headers=None, + ) + if hidden_params is not None: + for key in StandardLoggingHiddenParams.__annotations__.keys(): + if key in hidden_params: + if key == "additional_headers": + clean_hidden_params["additional_headers"] = ( + StandardLoggingPayloadSetup.get_additional_headers( + hidden_params[key] + ) + ) + else: + clean_hidden_params[key] = hidden_params[key] + return clean_hidden_params + def get_standard_logging_object_payload( kwargs: Optional[dict], @@ -2671,7 +2718,9 @@ def get_standard_logging_object_payload( if response_headers is not None: hidden_params = dict( StandardLoggingHiddenParams( - additional_headers=dict(response_headers), + additional_headers=StandardLoggingPayloadSetup.get_additional_headers( + dict(response_headers) + ), model_id=None, cache_key=None, api_base=None, @@ -2712,21 +2761,9 @@ def get_standard_logging_object_payload( ) ) # clean up litellm hidden params - clean_hidden_params = StandardLoggingHiddenParams( - model_id=None, - cache_key=None, - api_base=None, - response_cost=None, - additional_headers=None, + clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params( + hidden_params ) - if hidden_params is not None: - clean_hidden_params = StandardLoggingHiddenParams( - **{ # type: ignore - key: hidden_params[key] - for key in StandardLoggingHiddenParams.__annotations__.keys() - if key in hidden_params - } - ) # clean up litellm metadata clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata( metadata=metadata diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8f833c129cae..9578ed9eaeac 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -774,6 +774,20 @@ "supports_vision": true, "supports_prompt_caching": true }, + "azure/gpt-4o-mini-2024-07-18": { + "max_tokens": 16384, + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "input_cost_per_token": 0.000000165, + "output_cost_per_token": 0.00000066, + "cache_read_input_token_cost": 0.000000075, + "litellm_provider": "azure", + "mode": "chat", + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_vision": true, + "supports_prompt_caching": true + }, "azure/gpt-4-turbo-2024-04-09": { "max_tokens": 4096, "max_input_tokens": 128000, diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 5de5413eda7b..dc6bee9246f4 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -13,7 +13,7 @@ model_list: litellm_settings: fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }] - callbacks: ["otel"] + callbacks: ["otel", "prometheus"] router_settings: routing_strategy: latency-based-routing @@ -23,6 +23,6 @@ router_settings: # consider last five minutes of calls for latency calculation ttl: 300 - redis_host: os.environ/REDIS_HOST - redis_port: os.environ/REDIS_PORT - redis_password: os.environ/REDIS_PASSWORD \ No newline at end of file + # redis_host: os.environ/REDIS_HOST + # redis_port: os.environ/REDIS_PORT + # redis_password: os.environ/REDIS_PASSWORD \ No newline at end of file diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0b7a29c91bce..6658eb330263 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1433,12 +1433,19 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata): requester_metadata: Optional[dict] +class StandardLoggingAdditionalHeaders(TypedDict, total=False): + x_ratelimit_limit_requests: int + x_ratelimit_limit_tokens: int + x_ratelimit_remaining_requests: int + x_ratelimit_remaining_tokens: int + + class StandardLoggingHiddenParams(TypedDict): model_id: Optional[str] cache_key: Optional[str] api_base: Optional[str] response_cost: Optional[str] - additional_headers: Optional[dict] + additional_headers: Optional[StandardLoggingAdditionalHeaders] class StandardLoggingModelInformation(TypedDict): diff --git a/tests/logging_callback_tests/test_standard_logging_payload.py b/tests/logging_callback_tests/test_standard_logging_payload.py index f6599a005669..42d504a1e1b2 100644 --- a/tests/logging_callback_tests/test_standard_logging_payload.py +++ b/tests/logging_callback_tests/test_standard_logging_payload.py @@ -65,3 +65,42 @@ def test_get_usage(response_obj, expected_values): assert usage.prompt_tokens == expected_values[0] assert usage.completion_tokens == expected_values[1] assert usage.total_tokens == expected_values[2] + + +def test_get_additional_headers(): + additional_headers = { + "x-ratelimit-limit-requests": "2000", + "x-ratelimit-remaining-requests": "1999", + "x-ratelimit-limit-tokens": "160000", + "x-ratelimit-remaining-tokens": "160000", + "llm_provider-date": "Tue, 29 Oct 2024 23:57:37 GMT", + "llm_provider-content-type": "application/json", + "llm_provider-transfer-encoding": "chunked", + "llm_provider-connection": "keep-alive", + "llm_provider-anthropic-ratelimit-requests-limit": "2000", + "llm_provider-anthropic-ratelimit-requests-remaining": "1999", + "llm_provider-anthropic-ratelimit-requests-reset": "2024-10-29T23:57:40Z", + "llm_provider-anthropic-ratelimit-tokens-limit": "160000", + "llm_provider-anthropic-ratelimit-tokens-remaining": "160000", + "llm_provider-anthropic-ratelimit-tokens-reset": "2024-10-29T23:57:36Z", + "llm_provider-request-id": "req_01F6CycZZPSHKRCCctcS1Vto", + "llm_provider-via": "1.1 google", + "llm_provider-cf-cache-status": "DYNAMIC", + "llm_provider-x-robots-tag": "none", + "llm_provider-server": "cloudflare", + "llm_provider-cf-ray": "8da71bdbc9b57abb-SJC", + "llm_provider-content-encoding": "gzip", + "llm_provider-x-ratelimit-limit-requests": "2000", + "llm_provider-x-ratelimit-remaining-requests": "1999", + "llm_provider-x-ratelimit-limit-tokens": "160000", + "llm_provider-x-ratelimit-remaining-tokens": "160000", + } + additional_logging_headers = StandardLoggingPayloadSetup.get_additional_headers( + additional_headers + ) + assert additional_logging_headers == { + "x_ratelimit_limit_requests": 2000, + "x_ratelimit_remaining_requests": 1999, + "x_ratelimit_limit_tokens": 160000, + "x_ratelimit_remaining_tokens": 160000, + } From 50ff8c35b5b70ad1feb3dbbf878e5dc04171404b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 17:14:24 -0700 Subject: [PATCH 16/20] fix: fix linting error --- litellm/router_utils/cooldown_cache.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py index 44174f3b16c6..dbe767214a3e 100644 --- a/litellm/router_utils/cooldown_cache.py +++ b/litellm/router_utils/cooldown_cache.py @@ -17,13 +17,6 @@ else: Span = Any -if TYPE_CHECKING: - from opentelemetry.trace import Span as _Span - - Span = _Span -else: - Span = Any - class CooldownCacheValue(TypedDict): exception_received: str @@ -117,7 +110,6 @@ async def async_get_active_cooldowns( if results is None: return active_cooldowns - # Process the results for model_id, result in zip(model_ids, results): if result and isinstance(result, dict): From c72ea56719b779283feb1c6fb1998f8399a78f5a Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 17:24:23 -0700 Subject: [PATCH 17/20] fix(redis_cache.py): make sure ttl is always int (handle float values) Fixes issue where redis_client.ex was not working correctly due to float ttl --- litellm/caching/base_cache.py | 8 ++++++-- litellm/caching/redis_cache.py | 1 + litellm/proxy/_new_secret_config.yaml | 6 +++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/litellm/caching/base_cache.py b/litellm/caching/base_cache.py index 0699832ab307..a50e09bf97f5 100644 --- a/litellm/caching/base_cache.py +++ b/litellm/caching/base_cache.py @@ -23,8 +23,12 @@ def __init__(self, default_ttl: int = 60): self.default_ttl = default_ttl def get_ttl(self, **kwargs) -> Optional[int]: - if kwargs.get("ttl") is not None: - return kwargs.get("ttl") + kwargs_ttl: Optional[int] = kwargs.get("ttl") + if kwargs_ttl is not None: + try: + return int(kwargs_ttl) + except ValueError: + return self.default_ttl return self.default_ttl def set_cache(self, key, value, **kwargs): diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py index e6c408cc81d4..042a083a4095 100644 --- a/litellm/caching/redis_cache.py +++ b/litellm/caching/redis_cache.py @@ -301,6 +301,7 @@ async def async_set_cache(self, key, value, **kwargs): print_verbose( f"Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}" ) + try: if not hasattr(redis_client, "set"): raise Exception( diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index dc6bee9246f4..69a1119ccb94 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -23,6 +23,6 @@ router_settings: # consider last five minutes of calls for latency calculation ttl: 300 - # redis_host: os.environ/REDIS_HOST - # redis_port: os.environ/REDIS_PORT - # redis_password: os.environ/REDIS_PASSWORD \ No newline at end of file + redis_host: os.environ/REDIS_HOST + redis_port: os.environ/REDIS_PORT + redis_password: os.environ/REDIS_PASSWORD \ No newline at end of file From 3cec40a36878a7cd7e5215b6e6b731ac556bcda0 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 19:48:22 -0700 Subject: [PATCH 18/20] fix: fix linting error --- litellm/litellm_core_utils/litellm_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a35fbace7af6..4753779c0004 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2684,7 +2684,7 @@ def get_hidden_params( ) ) else: - clean_hidden_params[key] = hidden_params[key] + clean_hidden_params[key] = hidden_params[key] # type: ignore return clean_hidden_params From 498d2d9fcd1cf38c63461b7670a7a32d99f3fca6 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 20:51:17 -0700 Subject: [PATCH 19/20] test: update test --- litellm/integrations/prometheus.py | 14 +------------- tests/local_testing/test_caching.py | 4 ++-- .../test_prometheus_unit_tests.py | 9 +++++---- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 1ec2cc71cfa5..cbeb4d3366a4 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -871,25 +871,13 @@ def set_llm_deployment_success_metrics( if additional_headers := standard_logging_payload["hidden_params"][ "additional_headers" ]: + # OpenAI / OpenAI Compatible headers remaining_requests = additional_headers.get( "x_ratelimit_remaining_requests", None ) remaining_tokens = additional_headers.get( "x_ratelimit_remaining_tokens", None ) - # OpenAI / OpenAI Compatible headers - if ( - additional_headers - and "x_ratelimit_remaining_requests" in additional_headers - ): - remaining_requests = additional_headers[ - "x_ratelimit_remaining_requests" - ] - if ( - additional_headers - and "x_ratelimit_remaining_tokens" in additional_headers - ): - remaining_tokens = additional_headers["x_ratelimit_remaining_tokens"] if remaining_requests: """ diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index 3456f4535dd1..1116840b53d9 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -609,7 +609,7 @@ async def test_embedding_caching_redis_ttl(): type="redis", host="dummy_host", password="dummy_password", - default_in_redis_ttl=2.5, + default_in_redis_ttl=2, ) inputs = [ @@ -635,7 +635,7 @@ async def test_embedding_caching_redis_ttl(): print(f"redis pipeline set args: {args}") print(f"redis pipeline set kwargs: {kwargs}") assert kwargs.get("ex") == datetime.timedelta( - seconds=2.5 + seconds=2 ) # Check if TTL is set to 2.5 seconds diff --git a/tests/logging_callback_tests/test_prometheus_unit_tests.py b/tests/logging_callback_tests/test_prometheus_unit_tests.py index a2c49b35a1bc..494f83a654a7 100644 --- a/tests/logging_callback_tests/test_prometheus_unit_tests.py +++ b/tests/logging_callback_tests/test_prometheus_unit_tests.py @@ -549,13 +549,14 @@ def test_set_llm_deployment_success_metrics(prometheus_logger): standard_logging_payload = create_standard_logging_payload() + standard_logging_payload["hidden_params"]["additional_headers"] = { + "x_ratelimit_remaining_requests": 123, + "x_ratelimit_remaining_tokens": 4321, + } + # Create test data request_kwargs = { "model": "gpt-3.5-turbo", - "response_headers": { - "x-ratelimit-remaining-requests": 123, - "x-ratelimit-remaining-tokens": 4321, - }, "litellm_params": { "custom_llm_provider": "openai", "metadata": {"model_info": {"id": "model-123"}}, From 669cda3576f12df59ffe2237e0bc8748fa127bff Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 29 Oct 2024 22:00:13 -0700 Subject: [PATCH 20/20] fix: fix linting error --- litellm/router.py | 39 ++++++++++------------------ litellm/router_utils/handle_error.py | 29 ++++++++++++++++++++- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index ac26aa61e985..72c795530d2a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -81,7 +81,10 @@ run_async_fallback, run_sync_fallback, ) -from litellm.router_utils.handle_error import send_llm_exception_alert +from litellm.router_utils.handle_error import ( + async_raise_router_rate_limit_error, + send_llm_exception_alert, +) from litellm.router_utils.router_callbacks.track_deployment_metrics import ( increment_deployment_failures_for_current_minute, increment_deployment_successes_for_current_minute, @@ -5183,21 +5186,13 @@ async def async_get_available_deployment( ) if len(healthy_deployments) == 0: - if _allowed_model_region is None: - _allowed_model_region = "n/a" - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( - model_ids=model_ids, parent_otel_span=parent_otel_span - ) - _cooldown_list = _get_cooldown_deployments( - litellm_router_instance=self, parent_otel_span=parent_otel_span - ) - raise RouterRateLimitError( + rate_limit_error = await async_raise_router_rate_limit_error( + litellm_router_instance=self, model=model, - cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, - cooldown_list=_cooldown_list, + parent_otel_span=parent_otel_span, ) + raise rate_limit_error + start_time = time.time() if ( self.routing_strategy == "usage-based-routing-v2" @@ -5258,19 +5253,13 @@ async def async_get_available_deployment( verbose_router_logger.info( f"get_available_deployment for model: {model}, No deployment available" ) - model_ids = self.get_model_ids(model_name=model) - _cooldown_time = self.cooldown_cache.get_min_cooldown( - model_ids=model_ids, parent_otel_span=parent_otel_span - ) - _cooldown_list = await _async_get_cooldown_deployments( - litellm_router_instance=self, parent_otel_span=parent_otel_span - ) - raise RouterRateLimitError( + rate_limit_error = await async_raise_router_rate_limit_error( + litellm_router_instance=self, model=model, - cooldown_time=_cooldown_time, - enable_pre_call_checks=self.enable_pre_call_checks, - cooldown_list=_cooldown_list, + parent_otel_span=parent_otel_span, ) + raise rate_limit_error + verbose_router_logger.info( f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}" ) diff --git a/litellm/router_utils/handle_error.py b/litellm/router_utils/handle_error.py index 25b511027dd7..9428be15dd0d 100644 --- a/litellm/router_utils/handle_error.py +++ b/litellm/router_utils/handle_error.py @@ -1,15 +1,21 @@ import asyncio import traceback -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Optional +from litellm.router_utils.cooldown_handlers import _async_get_cooldown_deployments from litellm.types.integrations.slack_alerting import AlertType +from litellm.types.router import RouterRateLimitError if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + from litellm.router import Router as _Router LitellmRouter = _Router + Span = _Span else: LitellmRouter = Any + Span = Any async def send_llm_exception_alert( @@ -55,3 +61,24 @@ async def send_llm_exception_alert( alert_type=AlertType.llm_exceptions, alerting_metadata={}, ) + + +async def async_raise_router_rate_limit_error( + litellm_router_instance: LitellmRouter, + model: str, + parent_otel_span: Optional[Span], +): + model_ids = litellm_router_instance.get_model_ids(model_name=model) + _cooldown_time = litellm_router_instance.cooldown_cache.get_min_cooldown( + model_ids=model_ids, parent_otel_span=parent_otel_span + ) + _cooldown_list = await _async_get_cooldown_deployments( + litellm_router_instance=litellm_router_instance, + parent_otel_span=parent_otel_span, + ) + return RouterRateLimitError( + model=model, + cooldown_time=_cooldown_time, + enable_pre_call_checks=litellm_router_instance.enable_pre_call_checks, + cooldown_list=_cooldown_list, + )