Litellm dev 10 29 2024

docs/my-website/docs/proxy/cost_tracking.md

@@ -284,9 +284,7 @@ Output from script
 
 :::info
 
-Customer This is the value of `user_id` passed when calling [`/key/generate`](https://litellm-api.up.railway.app/#/key%20management/generate_key_fn_key_generate_post)
-
-[this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
+Customer [this is `user` passed to `/chat/completions` request](#how-to-track-spend-with-litellm)
 - [LiteLLM API key](virtual_keys.md)
 
 

litellm/caching/base_cache.py

@@ -23,8 +23,12 @@ def __init__(self, default_ttl: int = 60):
         self.default_ttl = default_ttl
 
     def get_ttl(self, **kwargs) -> Optional[int]:
-        if kwargs.get("ttl") is not None:
-            return kwargs.get("ttl")
+        kwargs_ttl: Optional[int] = kwargs.get("ttl")
+        if kwargs_ttl is not None:
+            try:
+                return int(kwargs_ttl)
+            except ValueError:
+                return self.default_ttl
         return self.default_ttl
 
     def set_cache(self, key, value, **kwargs):

litellm/caching/redis_cache.py

@@ -301,6 +301,7 @@ async def async_set_cache(self, key, value, **kwargs):
             print_verbose(
                 f"Set ASYNC Redis Cache: key: {key}\nValue {value}\nttl={ttl}"
             )
+
             try:
                 if not hasattr(redis_client, "set"):
                     raise Exception(

litellm/integrations/prometheus.py

@@ -849,9 +849,13 @@ def set_llm_deployment_success_metrics(
     ):
         try:
             verbose_logger.debug("setting remaining tokens requests metric")
-            standard_logging_payload: StandardLoggingPayload = request_kwargs.get(
-                "standard_logging_object", {}
+            standard_logging_payload: Optional[StandardLoggingPayload] = (
+                request_kwargs.get("standard_logging_object")
             )
+
+            if standard_logging_payload is None:
+                return
+
             model_group = standard_logging_payload["model_group"]
             api_base = standard_logging_payload["api_base"]
             _response_headers = request_kwargs.get("response_headers")
@@ -862,22 +866,18 @@ def set_llm_deployment_success_metrics(
             _model_info = _metadata.get("model_info") or {}
             model_id = _model_info.get("id", None)
 
-            remaining_requests = None
-            remaining_tokens = None
-            # OpenAI / OpenAI Compatible headers
-            if (
-                _response_headers
-                and "x-ratelimit-remaining-requests" in _response_headers
-            ):
-                remaining_requests = _response_headers["x-ratelimit-remaining-requests"]
-            if (
-                _response_headers
-                and "x-ratelimit-remaining-tokens" in _response_headers
-            ):
-                remaining_tokens = _response_headers["x-ratelimit-remaining-tokens"]
-            verbose_logger.debug(
-                f"remaining requests: {remaining_requests}, remaining tokens: {remaining_tokens}"
-            )
+            remaining_requests: Optional[int] = None
+            remaining_tokens: Optional[int] = None
+            if additional_headers := standard_logging_payload["hidden_params"][
+                "additional_headers"
+            ]:
+                # OpenAI / OpenAI Compatible headers
+                remaining_requests = additional_headers.get(
+                    "x_ratelimit_remaining_requests", None
+                )
+                remaining_tokens = additional_headers.get(
+                    "x_ratelimit_remaining_tokens", None
+                )
 
             if remaining_requests:
                 """

litellm/litellm_core_utils/core_helpers.py

@@ -80,7 +80,7 @@ def _get_parent_otel_span_from_kwargs(
 ) -> Union[Span, None]:
     try:
         if kwargs is None:
-            raise ValueError("kwargs is None")
+            return None
         litellm_params = kwargs.get("litellm_params")
         _metadata = kwargs.get("metadata") or {}
         if "litellm_parent_otel_span" in _metadata:

litellm/litellm_core_utils/litellm_logging.py

@@ -42,6 +42,7 @@
     ImageResponse,
     ModelResponse,
     StandardCallbackDynamicParams,
+    StandardLoggingAdditionalHeaders,
     StandardLoggingHiddenParams,
     StandardLoggingMetadata,
     StandardLoggingModelCostFailureDebugInformation,
@@ -2640,6 +2641,52 @@ def get_final_response_obj(
 
         return final_response_obj
 
+    @staticmethod
+    def get_additional_headers(
+        additiona_headers: Optional[dict],
+    ) -> Optional[StandardLoggingAdditionalHeaders]:
+
+        if additiona_headers is None:
+            return None
+
+        additional_logging_headers: StandardLoggingAdditionalHeaders = {}
+
+        for key in StandardLoggingAdditionalHeaders.__annotations__.keys():
+            _key = key.lower()
+            _key = _key.replace("_", "-")
+            if _key in additiona_headers:
+                try:
+                    additional_logging_headers[key] = int(additiona_headers[_key])  # type: ignore
+                except (ValueError, TypeError):
+                    verbose_logger.debug(
+                        f"Could not convert {additiona_headers[_key]} to int for key {key}."
+                    )
+        return additional_logging_headers
+
+    @staticmethod
+    def get_hidden_params(
+        hidden_params: Optional[dict],
+    ) -> StandardLoggingHiddenParams:
+        clean_hidden_params = StandardLoggingHiddenParams(
+            model_id=None,
+            cache_key=None,
+            api_base=None,
+            response_cost=None,
+            additional_headers=None,
+        )
+        if hidden_params is not None:
+            for key in StandardLoggingHiddenParams.__annotations__.keys():
+                if key in hidden_params:
+                    if key == "additional_headers":
+                        clean_hidden_params["additional_headers"] = (
+                            StandardLoggingPayloadSetup.get_additional_headers(
+                                hidden_params[key]
+                            )
+                        )
+                    else:
+                        clean_hidden_params[key] = hidden_params[key]  # type: ignore
+        return clean_hidden_params
+
 
 def get_standard_logging_object_payload(
     kwargs: Optional[dict],
@@ -2671,7 +2718,9 @@ def get_standard_logging_object_payload(
             if response_headers is not None:
                 hidden_params = dict(
                     StandardLoggingHiddenParams(
-                        additional_headers=dict(response_headers),
+                        additional_headers=StandardLoggingPayloadSetup.get_additional_headers(
+                            dict(response_headers)
+                        ),
                         model_id=None,
                         cache_key=None,
                         api_base=None,
@@ -2712,21 +2761,9 @@ def get_standard_logging_object_payload(
             )
         )
         # clean up litellm hidden params
-        clean_hidden_params = StandardLoggingHiddenParams(
-            model_id=None,
-            cache_key=None,
-            api_base=None,
-            response_cost=None,
-            additional_headers=None,
+        clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
+            hidden_params
         )
-        if hidden_params is not None:
-            clean_hidden_params = StandardLoggingHiddenParams(
-                **{  # type: ignore
-                    key: hidden_params[key]
-                    for key in StandardLoggingHiddenParams.__annotations__.keys()
-                    if key in hidden_params
-                }
-            )
         # clean up litellm metadata
         clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
             metadata=metadata

litellm/llms/vertex_ai_and_google_ai_studio/gemini/vertex_and_google_ai_studio_gemini.py

@@ -431,9 +431,13 @@ def _map_function(self, value: List[dict]) -> List[Tools]:
             elif openai_function_object is not None:
                 gtool_func_declaration = FunctionDeclaration(
                     name=openai_function_object["name"],
-                    description=openai_function_object.get("description", ""),
-                    parameters=openai_function_object.get("parameters", {}),
                 )
+                _description = openai_function_object.get("description", None)
+                _parameters = openai_function_object.get("parameters", None)
+                if _description is not None:
+                    gtool_func_declaration["description"] = _description
+                if _parameters is not None:
+                    gtool_func_declaration["parameters"] = _parameters
                 gtool_func_declarations.append(gtool_func_declaration)
             else:
                 # assume it's a provider-specific param

litellm/proxy/_new_secret_config.yaml

@@ -13,7 +13,7 @@ model_list:
 
 litellm_settings:
   fallbacks: [{ "claude-3-5-sonnet-20240620": ["claude-3-5-sonnet-aihubmix"] }]
-  callbacks: ["otel"]
+  callbacks: ["otel", "prometheus"]
 
 router_settings:
   routing_strategy: latency-based-routing

litellm/router.py

@@ -5255,6 +5255,7 @@ async def async_get_available_deployment(
                     parent_otel_span=parent_otel_span,
                 )
                 raise exception
+
             verbose_router_logger.info(
                 f"get_available_deployment for model: {model}, Selected deployment: {self.print_deployment(deployment)} for model: {model}"
             )

litellm/router_utils/handle_error.py

@@ -64,6 +64,7 @@ async def send_llm_exception_alert(
     )
 
 
+
 async def async_raise_no_deployment_exception(
     litellm_router_instance: LitellmRouter, model: str, parent_otel_span: Optional[Span]
 ):
@@ -73,6 +74,7 @@ async def async_raise_no_deployment_exception(
     verbose_router_logger.info(
         f"get_available_deployment for model: {model}, No deployment available"
     )
+
     model_ids = litellm_router_instance.get_model_ids(model_name=model)
     _cooldown_time = litellm_router_instance.cooldown_cache.get_min_cooldown(
         model_ids=model_ids, parent_otel_span=parent_otel_span

litellm/types/utils.py

@@ -1433,12 +1433,19 @@ class StandardLoggingMetadata(StandardLoggingUserAPIKeyMetadata):
     requester_metadata: Optional[dict]
 
 
+class StandardLoggingAdditionalHeaders(TypedDict, total=False):
+    x_ratelimit_limit_requests: int
+    x_ratelimit_limit_tokens: int
+    x_ratelimit_remaining_requests: int
+    x_ratelimit_remaining_tokens: int
+
+
 class StandardLoggingHiddenParams(TypedDict):
     model_id: Optional[str]
     cache_key: Optional[str]
     api_base: Optional[str]
     response_cost: Optional[str]
-    additional_headers: Optional[dict]
+    additional_headers: Optional[StandardLoggingAdditionalHeaders]
 
 
 class StandardLoggingModelInformation(TypedDict):

tests/llm_translation/test_optional_params.py

@@ -786,6 +786,7 @@ def test_unmapped_vertex_anthropic_model():
     assert "max_retries" not in optional_params
 
 
+
 @pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"])
 def test_anthropic_parallel_tool_calls(provider):
     optional_params = get_optional_params(

tests/llm_translation/test_vertex.py

@@ -12,8 +12,9 @@
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
-
+import pytest
 import litellm
+from litellm import get_optional_params
 
 
 def test_completion_pydantic_obj_2():
@@ -117,3 +118,63 @@ def test_build_vertex_schema():
     assert new_schema["type"] == schema["type"]
     assert new_schema["properties"] == schema["properties"]
     assert "required" in new_schema and new_schema["required"] == schema["required"]
+
+
+@pytest.mark.parametrize(
+    "tools, key",
+    [
+        ([{"googleSearchRetrieval": {}}], "googleSearchRetrieval"),
+        ([{"code_execution": {}}], "code_execution"),
+    ],
+)
+def test_vertex_tool_params(tools, key):
+
+    optional_params = get_optional_params(
+        model="gemini-1.5-pro",
+        custom_llm_provider="vertex_ai",
+        tools=tools,
+    )
+    print(optional_params)
+    assert optional_params["tools"][0][key] == {}
+
+
+@pytest.mark.parametrize(
+    "tool, expect_parameters",
+    [
+        (
+            {
+                "name": "test_function",
+                "description": "test_function_description",
+                "parameters": {
+                    "type": "object",
+                    "properties": {"test_param": {"type": "string"}},
+                },
+            },
+            True,
+        ),
+        (
+            {
+                "name": "test_function",
+            },
+            False,
+        ),
+    ],
+)
+def test_vertex_function_translation(tool, expect_parameters):
+    """
+    If param not set, don't set it in the request
+    """
+
+    tools = [tool]
+    optional_params = get_optional_params(
+        model="gemini-1.5-pro",
+        custom_llm_provider="vertex_ai",
+        tools=tools,
+    )
+    print(optional_params)
+    if expect_parameters:
+        assert "parameters" in optional_params["tools"][0]["function_declarations"][0]
+    else:
+        assert (
+            "parameters" not in optional_params["tools"][0]["function_declarations"][0]
+        )

tests/local_testing/test_caching.py

@@ -609,7 +609,7 @@ async def test_embedding_caching_redis_ttl():
             type="redis",
             host="dummy_host",
             password="dummy_password",
-            default_in_redis_ttl=2.5,
+            default_in_redis_ttl=2,
         )
 
         inputs = [
@@ -635,7 +635,7 @@ async def test_embedding_caching_redis_ttl():
             print(f"redis pipeline set args: {args}")
             print(f"redis pipeline set kwargs: {kwargs}")
             assert kwargs.get("ex") == datetime.timedelta(
-                seconds=2.5
+                seconds=2
             )  # Check if TTL is set to 2.5 seconds
 
 

tests/local_testing/test_function_calling.py

@@ -612,3 +612,34 @@ def test_passing_tool_result_as_list():
         print(resp)
 
     assert resp.usage.prompt_tokens_details.cached_tokens > 0
+
+
+def test_function_calling_with_gemini():
+    litellm.set_verbose = True
+    resp = litellm.completion(
+        model="gemini/gemini-1.5-pro-002",
+        messages=[
+            {
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "You are a helpful assistant that can interact with a computer to solve tasks.\n<IMPORTANT>\n* If user provides a path, you should NOT assume it's relative to the current working directory. Instead, you should explore the file system to find the file before working on it.\n</IMPORTANT>\n",
+                    }
+                ],
+                "role": "system",
+            },
+            {
+                "content": [{"type": "text", "text": "Hey, how's it going?"}],
+                "role": "user",
+            },
+        ],
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "finish",
+                    "description": "Finish the interaction when the task is complete OR if the assistant cannot proceed further with the task.",
+                },
+            },
+        ],
+    )