From 1c2041177ecf93c48c5f667d718baeecff4dcd3b Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 18 Jun 2026 21:49:39 -0700 Subject: [PATCH 1/5] refactor(completion): extract provider dispatch into typed helpers so basedpyright can analyze it completion packed a ~2,900-line per-provider if/elif dispatch into a single body, pushing it past basedpyright's code-flow complexity ceiling. basedpyright emitted "Code is too complex to analyze" and skipped the whole function, so every type error on the hottest request path was invisible and unguarded. This extracts each provider branch into its own helper that receives a single frozen _CompletionDispatchContext carrying the shared locals the dispatch reads (model, messages, optional_params, litellm_params, logging, headers, api_key, api_base, client, timeout, ...). Each helper destructures only what it uses and keeps its body verbatim, so the dispatch logic is unchanged; completion now sits well under the ceiling and basedpyright type-checks it and all 61 helpers. The context is built once after setup, right before the dispatch, and each arm becomes `response = _complete_(ctx)` feeding the existing single `return response`, preserving the original "dispatch sets response, return once" shape and the early-return/streaming semantics. The three deprecated no-op arms (clarifai, together_ai, palm) stay inline. Building the context surfaced a shadowing trap: acompletion, client, api_version, organization and text_completion are completion parameters that shadow module-level names, so a naive free-variable pass would have dropped them from the context and silently changed behavior; they are threaded through explicitly. Restoring analysis also surfaced get_secret()'s bool-inclusive return type broadening api_base in the anthropic and anthropic_text branches (it flows into .endswith()/+=); api_base is narrowed back to Optional[str] at those two sites. Genuinely dead assignments the monolith hid at function scope (an unused prompt build, two unused data dicts, and two no-op custom_llm_provider reassignments) are removed. No behavior change. The basedpyright per-rule budget is unchanged; the gate is confirmed by the CI run (the dev-only/local basedpyright env inflates reportMissingTypeStubs on any main.py edit, independent of this change, as the budget's own git history documents). --- litellm/main.py | 7773 +++++++++++++++++++++++++++-------------------- 1 file changed, 4539 insertions(+), 3234 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 63c5798e70af..b41a222bfe46 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -22,6 +22,7 @@ from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy +from dataclasses import dataclass from functools import partial from typing import ( TYPE_CHECKING, @@ -1085,3533 +1086,4837 @@ def _build_custom_pricing_entry( @tracer.wrap() -@client -def completion( # type: ignore - model: str, - # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create - messages: List = [], - timeout: Optional[Union[float, str, httpx.Timeout]] = None, - temperature: Optional[float] = None, - top_p: Optional[float] = None, - n: Optional[int] = None, - stream: Optional[bool] = None, - stream_options: Optional[dict] = None, - stop=None, - max_completion_tokens: Optional[int] = None, - max_tokens: Optional[int] = None, - modalities: Optional[List[ChatCompletionModality]] = None, - prediction: Optional[ChatCompletionPredictionContentParam] = None, - audio: Optional[ChatCompletionAudioParam] = None, - presence_penalty: Optional[float] = None, - frequency_penalty: Optional[float] = None, - logit_bias: Optional[dict] = None, - user: Optional[str] = None, - # openai v1.0+ new params - reasoning_effort: Optional[ - Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] - ] = None, - verbosity: Optional[Literal["low", "medium", "high"]] = None, - response_format: Optional[Union[dict, Type[BaseModel]]] = None, - seed: Optional[int] = None, - tools: Optional[List] = None, - tool_choice: Optional[Union[str, dict]] = None, - logprobs: Optional[bool] = None, - top_logprobs: Optional[int] = None, - parallel_tool_calls: Optional[bool] = None, - web_search_options: Optional[OpenAIWebSearchOptions] = None, - include_server_side_tool_invocations: Optional[bool] = None, - deployment_id=None, - extra_headers: Optional[dict] = None, - safety_identifier: Optional[str] = None, - service_tier: Optional[str] = None, - # soon to be deprecated params by OpenAI - functions: Optional[List] = None, - function_call: Optional[str] = None, - # set api_base, api_version, api_key - base_url: Optional[str] = None, - api_version: Optional[str] = None, - api_key: Optional[str] = None, - model_list: Optional[list] = None, # pass in a list of api_base,keys, etc. - # Optional liteLLM function params - thinking: Optional[AnthropicThinkingParam] = None, - # Session management - shared_session: Optional["ClientSession"] = None, - # Per-request JSON schema validation (overrides litellm.enable_json_schema_validation) - enable_json_schema_validation: Optional[bool] = None, - **kwargs, -) -> Union[ModelResponse, CustomStreamWrapper]: - """ - Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly) - Parameters: - model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/ - messages (List): A list of message objects representing the conversation context (default is an empty list). +@dataclass(frozen=True, slots=True) +class _CompletionDispatchContext: + _azure_detection_model: str + acompletion: bool + api_base: Optional[str] + api_key: Optional[str] + api_version: Optional[str] + client: Any + custom_llm_provider: str + custom_prompt_dict: dict + extra_headers: Optional[dict] + headers: dict + hf_model_name: Optional[str] + kwargs: dict + litellm_params: dict + logger_fn: Optional[Callable] + logging: LiteLLMLoggingObj + max_retries: Optional[int] + max_tokens: Optional[int] + messages: List + metadata: Optional[dict] + model: str + model_response: ModelResponse + optional_params: dict + organization: Optional[str] + provider_config: Optional[BaseConfig] + shared_session: Optional["ClientSession"] + stream: Optional[bool] + temperature: Optional[float] + text_completion: bool + timeout: Optional[Union[float, str, httpx.Timeout]] + top_p: Optional[float] + + +def _complete_azure(ctx: _CompletionDispatchContext): + _azure_detection_model = ctx._azure_detection_model + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + max_retries = ctx.max_retries + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + dynamic_params = False + if client is not None and ( + isinstance(client, openai.AzureOpenAI) + or isinstance(client, openai.AsyncAzureOpenAI) + ): + dynamic_params = _check_dynamic_azure_params( + azure_client_params={"api_version": api_version}, + azure_client=client, + ) - OPTIONAL PARAMS - functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list). - function_call (str, optional): The name of the function to call within the conversation (default is an empty string). - temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0). - top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). - n (int, optional): The number of completions to generate (default is 1). - stream (bool, optional): If True, return a streaming response (default is False). - stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true. - stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. - max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). - max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. - modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request.. You can use `["text", "audio"]` - prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content. - audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"] - presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. - frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far. - logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion. - user (str, optional): A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse. - logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message - top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. - metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. - api_base (str, optional): Base URL for the API (default is None). - api_version (str, optional): API version (default is None). - api_key (str, optional): API key (default is None). - model_list (list, optional): List of api base, version, keys - extra_headers (dict, optional): Additional headers to include in the request. + api_type = get_secret("AZURE_API_TYPE") or "azure" - LITELLM Specific Params - mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). - custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock" - max_retries (int, optional): The number of retries to attempt (default is 0). - Returns: - ModelResponse: A response object containing the generated completion and associated metadata. + api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") - Note: - - This function is used to perform completions() using the specified language model. - - It supports various optional parameters for customizing the completion behavior. - - If 'mock_response' is provided, a mock completion response is returned for testing or debugging. - """ - ### VALIDATE Request ### - if model is None: - raise ValueError("model param not passed in.") - # validate messages - messages = validate_and_fix_openai_messages(messages=messages) - tools = validate_and_fix_openai_tools(tools=tools) - # validate tool_choice - tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) - # validate optional params - stop = validate_openai_optional_params(stop=stop) - # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens) - thinking = validate_and_fix_thinking_param(thinking=thinking) + api_version = ( + api_version + or litellm.api_version + or get_secret_str("AZURE_API_VERSION") + or litellm.AZURE_DEFAULT_API_VERSION + ) - ######### unpacking kwargs ##################### - args = locals() + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) - skip_mcp_handler = kwargs.pop("_skip_mcp_handler", False) - if not skip_mcp_handler and tools: - from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp - from litellm.responses.mcp.litellm_proxy_mcp_handler import ( - LiteLLM_Proxy_MCP_Handler, + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret_str("AZURE_AD_TOKEN") + + azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + if max_retries is not None: + optional_params["max_retries"] = max_retries + + if litellm.AzureOpenAIO1Config().is_o_series_model(model=_azure_detection_model): + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIO1Config.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + response = azure_o1_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + custom_llm_provider=custom_llm_provider, ) - from litellm.types.llms.openai import ToolParam + else: + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=api_version, + api_type=api_type, + dynamic_params=dynamic_params, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, # type: ignore + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response + + +def _complete_azure_text(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + api_version = ctx.api_version + client = ctx.client + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_type = get_secret_str("AZURE_API_TYPE") or "azure" + + api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + + if api_base is None: + raise ValueError( + "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." + ) + + api_version = ( + api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION") + ) + + api_key = ( + api_key + or litellm.api_key + or litellm.azure_key + or get_secret_str("AZURE_OPENAI_API_KEY") + or get_secret_str("AZURE_API_KEY") + ) + + azure_ad_token = optional_params.get("extra_body", {}).pop( + "azure_ad_token", None + ) or get_secret_str("AZURE_AD_TOKEN") + + azure_ad_token_provider = litellm_params.get("azure_ad_token_provider", None) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## LOAD CONFIG - if set + config = litellm.AzureOpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + response = azure_text_completions.completion( + model=model, + messages=messages, + headers=headers, + api_key=api_key, + api_base=api_base, + api_version=cast(str, api_version), + api_type=api_type, + azure_ad_token=azure_ad_token, + azure_ad_token_provider=azure_ad_token_provider, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, # pass AsyncAzureOpenAI, AzureOpenAI client + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={ + "headers": headers, + "api_version": api_version, + "api_base": api_base, + }, + ) + + return response + + +def _complete_deepseek(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_azure_ai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + + azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model) + + # Check if this is an agents route - model format: azure_ai/agents/ + if azure_ai_route == "agents": + from litellm.llms.azure_ai.agents import AzureAIAgentsConfig + + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure AI Agents requests require an api_base. " + "Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + response = AzureAIAgentsConfig.completion( + model=model, + messages=messages, + api_base=api_base, + api_key=api_key, + model_response=model_response, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + acompletion=acompletion, + stream=stream, + headers=headers or litellm.headers, + ) + + # Check if this is a Claude model - route to Azure Anthropic handler + elif "claude" in model.lower(): + # Use Azure Anthropic handler for Claude models + api_base = AzureFoundryModelInfo.get_api_base(api_base) + if api_base is None: + raise ValueError( + "Azure Anthropic requests require an api_base. " + "Set `api_base` or the AZURE_AI_API_BASE env var." + ) + api_key = AzureFoundryModelInfo.get_api_key(api_key) + + # Ensure the URL ends with /v1/messages for Anthropic + if api_base: + api_base = api_base.rstrip("/") + if not api_base.endswith("/v1/messages"): + if "/anthropic" in api_base: + parts = api_base.split("/anthropic", 1) + api_base = parts[0] + "/anthropic" + else: + api_base = api_base + "/anthropic" + api_base = api_base + "/v1/messages" + + response = azure_anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + else: + # Non-Claude models use standard Azure AI flow + api_base = AzureFoundryModelInfo.get_api_base(api_base) + # set API KEY + api_key = AzureFoundryModelInfo.get_api_key(api_key) - # Check if MCP tools are present (following responses pattern) - # Cast tools to Optional[Iterable[ToolParam]] for type checking - tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools) - if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( - tools=tools_for_mcp - ): - # Return coroutine - acompletion will await it - # completion() can return a coroutine when MCP tools are present, which acompletion() awaits - return acompletion_with_mcp( # type: ignore[return-value] + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + ## FOR COHERE + if "command-r" in model: # make sure tool call in messages are str + messages = stringify_json_tool_call_content(messages=messages) + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( model=model, messages=messages, - functions=functions, - function_call=function_call, - timeout=timeout, - temperature=temperature, - top_p=top_p, - n=n, - stream=stream, - stream_options=stream_options, - stop=stop, - max_tokens=max_tokens, - max_completion_tokens=max_completion_tokens, - modalities=modalities, - prediction=prediction, - audio=audio, - presence_penalty=presence_penalty, - frequency_penalty=frequency_penalty, - logit_bias=logit_bias, - user=user, - response_format=response_format, - seed=seed, - tools=tools, - tool_choice=tool_choice, - parallel_tool_calls=parallel_tool_calls, - logprobs=logprobs, - top_logprobs=top_logprobs, - deployment_id=deployment_id, - reasoning_effort=reasoning_effort, - verbosity=verbosity, - safety_identifier=safety_identifier, - service_tier=service_tier, - base_url=base_url, - api_version=api_version, + headers=headers, + model_response=model_response, api_key=api_key, - model_list=model_list, - extra_headers=extra_headers, - thinking=thinking, - web_search_options=web_search_options, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, shared_session=shared_session, - enable_json_schema_validation=enable_json_schema_validation, - **kwargs, + timeout=timeout, # type: ignore + client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, ) - api_base = kwargs.get("api_base", None) - mock_response: Optional[MOCK_RESPONSE_TYPE] = kwargs.get("mock_response", None) - mock_tool_calls = kwargs.get("mock_tool_calls", None) - mock_timeout = cast(Optional[bool], kwargs.get("mock_timeout", None)) - force_timeout = kwargs.get("force_timeout", 600) ## deprecated - logger_fn = kwargs.get("logger_fn", None) - verbose = kwargs.get("verbose", False) - custom_llm_provider = kwargs.get("custom_llm_provider", None) - litellm_logging_obj = kwargs.get("litellm_logging_obj", None) - id = kwargs.get("id", None) - metadata = kwargs.get("metadata", None) - model_info = kwargs.get("model_info", None) - proxy_server_request = kwargs.get("proxy_server_request", None) - fallbacks = kwargs.get("fallbacks", None) - provider_specific_header = cast( - Optional[ProviderSpecificHeader], kwargs.get("provider_specific_header", None) - ) - headers = kwargs.get("headers", None) or extra_headers - - ensure_alternating_roles: Optional[bool] = kwargs.get( - "ensure_alternating_roles", None - ) - user_continue_message: Optional[ChatCompletionUserMessage] = kwargs.get( - "user_continue_message", None - ) - assistant_continue_message: Optional[ChatCompletionAssistantMessage] = kwargs.get( - "assistant_continue_message", None - ) - if headers is None: - headers = {} - if extra_headers is not None: - headers.update(extra_headers) - # Inject proxy auth headers if configured - if litellm.proxy_auth is not None: - try: - proxy_headers = litellm.proxy_auth.get_auth_headers() - headers.update(proxy_headers) except Exception as e: - verbose_logger.warning(f"Failed to get proxy auth headers: {e}") - num_retries = kwargs.get( - "num_retries", None - ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. - max_retries = kwargs.get("max_retries", None) - cooldown_time = kwargs.get("cooldown_time", None) - context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None) - organization = kwargs.get("organization", None) - ### VERIFY SSL ### - ssl_verify = kwargs.get("ssl_verify", None) - ### CUSTOM MODEL COST ### - input_cost_per_token = kwargs.get("input_cost_per_token", None) - output_cost_per_token = kwargs.get("output_cost_per_token", None) - input_cost_per_second = kwargs.get("input_cost_per_second", None) - output_cost_per_second = kwargs.get("output_cost_per_second", None) - ### CUSTOM PROMPT TEMPLATE ### - initial_prompt_value = kwargs.get("initial_prompt_value", None) - roles = kwargs.get("roles", None) - final_prompt_value = kwargs.get("final_prompt_value", None) - bos_token = kwargs.get("bos_token", None) - eos_token = kwargs.get("eos_token", None) - preset_cache_key = kwargs.get("preset_cache_key", None) - hf_model_name = kwargs.get("hf_model_name", None) - supports_system_message = kwargs.get("supports_system_message", None) - base_model = kwargs.get("base_model", None) or ( - model_info.get("base_model") if isinstance(model_info, dict) else None + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def _complete_text_completion_openai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + text_completion = ctx.text_completion + timeout = ctx.timeout + + openai.api_type = "openai" + + api_base = ( + api_base + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" ) - ### DISABLE FLAGS ### - disable_add_transform_inline_image_block = kwargs.get( - "disable_add_transform_inline_image_block", None + + openai.api_version = None + # set API KEY + + api_key = ( + api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") ) - ### TEXT COMPLETION CALLS ### - text_completion = kwargs.get("text_completion", False) - atext_completion = kwargs.get("atext_completion", False) - ### ASYNC CALLS ### - acompletion = kwargs.get("acompletion", False) - client = kwargs.get("client", None) - ### Admin Controls ### - no_log = kwargs.get("no-log", False) - ### PROMPT MANAGEMENT ### - prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) - prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) - litellm_system_prompt = kwargs.get("litellm_system_prompt", None) - ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 - messages = get_completion_messages( + + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.OpenAITextCompletionConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + if litellm.organization: + openai.organization = litellm.organization + + ## COMPLETION CALL + _response = openai_text_completions.completion( + model=model, messages=messages, - ensure_alternating_roles=ensure_alternating_roles or False, - user_continue_message=user_continue_message, - assistant_continue_message=assistant_continue_message, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + acompletion=acompletion, + client=client, # pass AsyncOpenAI, OpenAI client + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore ) - ######## end of unpacking kwargs ########### - non_default_params = get_non_default_completion_params(kwargs=kwargs) - litellm_params = {} # used to prevent unbound var errors - ## PROMPT MANAGEMENT HOOKS ## - if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( - litellm_logging_obj.should_run_prompt_management_hooks( - prompt_id=prompt_id, non_default_params=non_default_params - ) + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False ): - ( - model, - messages, - optional_params, - ) = litellm_logging_obj.get_chat_completion_prompt( - model=model, - messages=messages, - non_default_params=non_default_params, - prompt_id=prompt_id, - prompt_variables=prompt_variables, - prompt_label=kwargs.get("prompt_label", None), - prompt_version=kwargs.get("prompt_version", None), + # convert to chat completion response + _response = ( + litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) ) - ### LITELLM SYSTEM PROMPT ### - if litellm_system_prompt: - messages = add_system_prompt_to_messages( - messages=messages, - system_prompt=litellm_system_prompt, - merge_with_first_system=True, + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, ) + response = _response + + return response + + +def _complete_fireworks_ai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout try: - if base_url is not None: - api_base = base_url - if num_retries is not None: - max_retries = num_retries - logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj) - fallbacks = fallbacks or litellm.model_fallbacks - if fallbacks is not None: - return completion_with_fallbacks(**args) - if model_list is not None: - deployments = [ - m["litellm_params"] for m in model_list if m["model_name"] == model - ] - return litellm.batch_completion_models(deployments=deployments, **args) - if litellm.model_alias_map and model in litellm.model_alias_map: - model = litellm.model_alias_map[ - model - ] # update the model to the actual value if an alias has been passed in - model_response = ModelResponse() - setattr(model_response, "usage", litellm.Usage()) - if ( - kwargs.get("azure", False) is True - ): # don't remove flag check, to remain backwards compatible for repos like Codium - custom_llm_provider = "azure" - if deployment_id is not None: # azure llms - model = deployment_id - custom_llm_provider = "azure" - _supplemental_provider_params = { - k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs - } - model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + return response + + +def _complete_heroku(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( model=model, - custom_llm_provider=custom_llm_provider, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, api_key=api_key, - litellm_params=( - GenericLiteLLMParams(**_supplemental_provider_params) - if _supplemental_provider_params - else None - ), + original_response=str(e), + additional_args={"headers": headers}, ) + raise e - ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name - responses_api_model_info, model = responses_api_bridge_check( + return response + + +def _complete_ragflow(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + try: + response = base_llm_http_handler.completion( model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, ) + raise e - if not _should_allow_input_examples( - custom_llm_provider=custom_llm_provider, model=model - ): - tools = _drop_input_examples_from_tools(tools=tools) + return response - if provider_specific_header is not None: - headers.update( - ProviderSpecificHeaderUtils.get_provider_specific_headers( - provider_specific_header=provider_specific_header, - custom_llm_provider=custom_llm_provider, - ) - ) - if model_response is not None and hasattr(model_response, "_hidden_params"): - model_response._hidden_params["custom_llm_provider"] = custom_llm_provider - model_response._hidden_params["region_name"] = kwargs.get( - "aws_region_name", None - ) # support region-based pricing for bedrock +def _complete_xai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout - ### TIMEOUT LOGIC ### - timeout = CompletionTimeout.resolve( - timeout, - kwargs, - custom_llm_provider, - global_timeout=getattr(litellm, "request_timeout", None), - supports_httpx_timeout=supports_httpx_timeout, + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e - ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### - if ( - input_cost_per_token is not None and output_cost_per_token is not None - ) or input_cost_per_second is not None: - litellm.register_model( - { - f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - model_info=model_info, - ) - } - ) - ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### - custom_prompt_dict = {} # type: ignore - if ( - initial_prompt_value - or roles - or final_prompt_value - or bos_token - or eos_token - ): - custom_prompt_dict = {model: {}} - if initial_prompt_value: - custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value - if roles: - custom_prompt_dict[model]["roles"] = roles - if final_prompt_value: - custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value - if bos_token: - custom_prompt_dict[model]["bos_token"] = bos_token - if eos_token: - custom_prompt_dict[model]["eos_token"] = eos_token + return response - messages = update_messages_with_model_file_ids( - messages=messages, - model_id=kwargs.get("model_info", {}).get("id", None), - model_file_id_mapping=cast( - Dict[str, Dict[str, str]], - kwargs.get("model_file_id_mapping") or {}, - ), - ) - provider_config: Optional[BaseConfig] = None - if custom_llm_provider is not None and custom_llm_provider in [ - provider.value for provider in LlmProviders - ]: - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders(custom_llm_provider), - base_model=base_model, - ) +def _complete_groq(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("GROQ_API_BASE") + or "https://api.groq.com/openai/v1" + ) - if provider_config is not None: - messages = provider_config.translate_developer_role_to_system_role( - messages=messages - ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.groq_key + or get_secret("GROQ_API_KEY") + ) + headers = headers or litellm.headers + + ## LOAD CONFIG - if set + config = litellm.GroqChatConfig.get_config() + for k, v in config.items(): if ( - supports_system_message is not None - and isinstance(supports_system_message, bool) - and supports_system_message is False - ): - messages = map_system_message_pt(messages=messages) + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v - if dynamic_api_key is not None: - api_key = dynamic_api_key - # check if user passed in any of the OpenAI optional params - optional_param_args = { - "functions": functions, - "function_call": function_call, - "temperature": temperature, - "top_p": top_p, - "n": n, - "stream": stream, - "stream_options": stream_options, - "stop": stop, - "max_tokens": max_tokens, - "max_completion_tokens": max_completion_tokens, - "modalities": modalities, - "prediction": prediction, - "audio": audio, - "presence_penalty": presence_penalty, - "frequency_penalty": frequency_penalty, - "logit_bias": logit_bias, - "user": user, - # params to identify the model - "model": model, - "custom_llm_provider": custom_llm_provider, - "response_format": response_format, - "seed": seed, - "tools": tools, - "tool_choice": tool_choice, - "max_retries": max_retries, - "logprobs": logprobs, - "top_logprobs": top_logprobs, - "api_version": api_version, - "parallel_tool_calls": parallel_tool_calls, - "messages": messages, - "reasoning_effort": reasoning_effort, - "thinking": thinking, - "web_search_options": web_search_options, - "include_server_side_tool_invocations": ( - include_server_side_tool_invocations - if include_server_side_tool_invocations is not None - else kwargs.get("include_server_side_tool_invocations") - ), - "safety_identifier": safety_identifier, - "service_tier": service_tier, - "allowed_openai_params": kwargs.get("allowed_openai_params"), - "base_model": base_model, - } - optional_params = get_optional_params( - **optional_param_args, **non_default_params - ) - processed_non_default_params = pre_process_non_default_params( - model=model, - passed_params=optional_param_args, - special_params=non_default_params, - custom_llm_provider=custom_llm_provider, - additional_drop_params=kwargs.get("additional_drop_params"), - remove_sensitive_keys=True, - add_provider_specific_params=True, - provider_config=provider_config, + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + + return response + + +def _complete_bedrock_mantle(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") + api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") + headers = headers or litellm.headers + config = litellm.BedrockMantleChatConfig.get_config() + for k, v in config.items(): + if k not in optional_params: + optional_params[k] = v + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + return response + + +def _complete_a2a(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + ( + api_base, + api_key, + headers, + ) = litellm.A2AConfig.resolve_agent_config_from_registry( + model=model, + api_base=api_base, + api_key=api_key, + headers=headers, + optional_params=optional_params, + ) + + # Fall back to environment variables and defaults + api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") + + if api_base is None: + raise Exception( + "api_base is required for A2A provider. " + "Either provide api_base parameter, set A2A_API_BASE environment variable, " + "or register the agent in the proxy with model='a2a/'." ) - if litellm.add_function_to_prompt and optional_params.get( - "functions_unsupported_model", None - ): # if user opts to add it to prompt, when API doesn't support function calling - functions_unsupported_model = optional_params.pop( - "functions_unsupported_model" - ) - messages = function_call_prompt( - messages=messages, functions=functions_unsupported_model - ) + headers = headers or litellm.headers + + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + + return response - # For logging - save the values of the litellm-specific params passed in - litellm_params = get_litellm_params( - acompletion=acompletion, + +def _complete_gigachat(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.api_key + or litellm.gigachat_key + or get_secret("GIGACHAT_API_KEY") + or get_secret("GIGACHAT_CREDENTIALS") + ) + + headers = headers or litellm.headers or {} + + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, api_key=api_key, - force_timeout=force_timeout, - logger_fn=logger_fn, - verbose=verbose, - custom_llm_provider=custom_llm_provider, api_base=api_base, - litellm_call_id=kwargs.get("litellm_call_id", None), - model_alias_map=litellm.model_alias_map, - completion_call_id=id, - metadata=metadata, - model_info=model_info, - proxy_server_request=proxy_server_request, - preset_cache_key=preset_cache_key, - no_log=no_log, - input_cost_per_second=input_cost_per_second, - input_cost_per_token=input_cost_per_token, - output_cost_per_second=output_cost_per_second, - output_cost_per_token=output_cost_per_token, - cooldown_time=cooldown_time, - text_completion=kwargs.get("text_completion"), - azure_ad_token_provider=kwargs.get("azure_ad_token_provider"), - user_continue_message=kwargs.get("user_continue_message"), - base_model=base_model, - litellm_trace_id=kwargs.get("litellm_trace_id"), - litellm_session_id=kwargs.get("litellm_session_id"), - hf_model_name=hf_model_name, - custom_prompt_dict=custom_prompt_dict, - litellm_metadata=kwargs.get("litellm_metadata"), - disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, - drop_params=kwargs.get("drop_params"), - prompt_id=prompt_id, - prompt_variables=prompt_variables, - ssl_verify=ssl_verify, - merge_reasoning_content_in_choices=kwargs.get( - "merge_reasoning_content_in_choices", None - ), - use_litellm_proxy=kwargs.get("use_litellm_proxy", False), - api_version=api_version, - azure_ad_token=kwargs.get("azure_ad_token"), - tenant_id=kwargs.get("tenant_id"), - client_id=kwargs.get("client_id"), - client_secret=kwargs.get("client_secret"), - azure_username=kwargs.get("azure_username"), - azure_password=kwargs.get("azure_password"), - azure_scope=kwargs.get("azure_scope"), - max_retries=max_retries, - timeout=timeout, - litellm_request_debug=kwargs.get("litellm_request_debug", False), - tpm=kwargs.get("tpm"), - rpm=kwargs.get("rpm"), - use_xai_oauth=kwargs.get("use_xai_oauth", False), - aws_bedrock_project_id=kwargs.get("aws_bedrock_project_id"), - ) - cast(LiteLLMLoggingObj, logging).update_environment_variables( - model=model, - user=user, - optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, ) - if mock_response or mock_tool_calls or mock_timeout: - kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered - return mock_completion( - model, - messages, - stream=stream, - n=n, - mock_response=mock_response, - mock_tool_calls=mock_tool_calls, - logging=logging, - acompletion=acompletion, - mock_delay=kwargs.get("mock_delay", None), - custom_llm_provider=custom_llm_provider, - mock_timeout=mock_timeout, - timeout=timeout, - ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e - ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map - # Only run the second bridge check if the first one didn't already - # detect responses mode (e.g. via the "responses/" prefix). The second - # check handles cases like gpt-5.4+ with tools+reasoning_effort or - # reasoningSummary/reasoning_summary without tools (AI SDK) that the first - # (early) check doesn't cover. - _reasoning_summary_for_bridge = peek_reasoning_summary_aliases(optional_params) - if responses_api_model_info.get("mode") != "responses": - responses_api_model_info, model = responses_api_bridge_check( - model=model, - custom_llm_provider=custom_llm_provider, - web_search_options=web_search_options, - tools=tools, - reasoning_effort=reasoning_effort, - reasoning_summary=_reasoning_summary_for_bridge, - ) + return response - # Use base_model (the true underlying model) for Azure model-type - # detection when the deployment name differs from the model name. - _azure_detection_model = base_model or model - if responses_api_model_info.get("mode") == "responses": - from litellm.completion_extras import responses_api_bridge +def _complete_sap(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + headers = headers or litellm.headers + ## LOAD CONFIG - if set + config = litellm.GenAIHubOrchestrationConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v - optional_params, rs_val = ( - strip_reasoning_summary_aliases_from_optional_params(optional_params) - ) + response = sap_gen_ai_hub_chat_completions.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + shared_session=shared_session, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + api_key=api_key, + api_base=api_base, + stream=stream, + ) - if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: - optional_params["reasoning_effort"] = reasoning_effort - elif rs_val is not None: - eff = optional_params.get("reasoning_effort", reasoning_effort) - if isinstance(eff, dict): - optional_params["reasoning_effort"] = {**eff, "summary": rs_val} - elif eff is not None: - optional_params["reasoning_effort"] = { - "effort": eff, - "summary": rs_val, - } - else: - optional_params["reasoning_effort"] = {"summary": rs_val} + return response - return responses_api_bridge.completion( + +def _complete_aiohttp_openai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + response = base_llm_aiohttp_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + return response + + +def _complete_cometapi(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.cometapi_key + or get_secret_str("COMETAPI_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COMETAPI_API_BASE") + or "https://api.cometapi.com/v1" + ) + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) + + ## LOGGING + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_minimax(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("MINIMAX_API_BASE") + or "https://api.minimax.io/v1" + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_hosted_vllm(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + logging.post_call(input=messages, api_key=api_key, original_response=response) + + return response + + +def _complete_custom_openai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + metadata = ctx.metadata + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + organization = ctx.organization + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or get_secret("OPENAI_BASE_URL") + or get_secret("OPENAI_API_BASE") + or "https://api.openai.com/v1" + ) + organization = ( + organization + or litellm.organization + or get_secret("OPENAI_ORGANIZATION") + or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 + ) + openai.organization = organization + # set API KEY + api_key = ( + api_key + or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there + or litellm.openai_key + or get_secret("OPENAI_API_KEY") + ) + + headers = headers or litellm.headers + + # Add GitHub Copilot headers (same as /responses endpoint does) + if custom_llm_provider == "github_copilot": + from litellm.llms.github_copilot.authenticator import Authenticator + from litellm.llms.github_copilot.common_utils import ( + get_copilot_default_headers, + ) + + copilot_auth = Authenticator() + copilot_api_key = copilot_auth.get_api_key() + copilot_headers = get_copilot_default_headers(copilot_api_key) + if extra_headers: + copilot_headers.update(extra_headers) + extra_headers = copilot_headers + + if extra_headers is not None: + optional_params["extra_headers"] = extra_headers + + if ( + litellm.enable_preview_features and metadata is not None + ): # [PREVIEW] allow metadata to be passed to OPENAI + openai_metadata = get_requester_metadata(metadata) + if openai_metadata is not None: + optional_params["metadata"] = openai_metadata + + ## LOAD CONFIG - if set + config = litellm.OpenAIConfig.get_config() + for k, v in config.items(): + if ( + k not in optional_params + ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in + optional_params[k] = v + + ## COMPLETION CALL + use_base_llm_http_handler = get_secret_bool( + "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER" + ) + + try: + if use_base_llm_http_handler: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + else: + response = openai_chat_completions.completion( model=model, messages=messages, headers=headers, model_response=model_response, + print_verbose=print_verbose, api_key=api_key, api_base=api_base, acompletion=acompletion, logging_obj=logging, optional_params=optional_params, litellm_params=litellm_params, + logger_fn=logger_fn, timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client + organization=organization, custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - elif ( - custom_llm_provider == "openai" - and OpenAIGPT5Config.is_model_gpt_5_model(model) - ) or ( - custom_llm_provider == "azure" - and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model( - _azure_detection_model - ) - ): - optional_params, _ = strip_reasoning_summary_aliases_from_optional_params( - optional_params + shared_session=shared_session, ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e + + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) + + return response + + +def _complete_mistral(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") + api_base = ( + api_base + or litellm.api_base + or get_secret("MISTRAL_API_BASE") + or "https://api.mistral.ai/v1" + ) + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + model_response=model_response, + encoding=_get_encoding(), + logging_obj=logging, + optional_params=optional_params, + timeout=timeout, + litellm_params=litellm_params, + shared_session=shared_session, + acompletion=acompletion, + stream=stream, + api_key=api_key, + headers=headers, + client=client, + provider_config=provider_config, + ) + + return response + + +def _complete_replicate(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + replicate_key = ( + api_key + or litellm.replicate_key + or litellm.api_key + or get_secret("REPLICATE_API_KEY") + or get_secret("REPLICATE_API_TOKEN") + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("REPLICATE_API_BASE") + or "https://api.replicate.com/v1" + ) + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + model_response = replicate_chat_completion( # type: ignore + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), # for calculating input/output tokens + api_key=replicate_key, + logging_obj=logging, + custom_prompt_dict=custom_prompt_dict, + acompletion=acompletion, + headers=headers, + ) + + if optional_params.get("stream", False) is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=replicate_key, + original_response=model_response, + ) + + response = model_response + + return response + + +def _complete_anthropic_text(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + api_base = cast( + Optional[str], + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/complete", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if ( + api_base is not None + and not disable_url_suffix + and not api_base.endswith("/v1/complete") + ): + api_base += "/v1/complete" + elif disable_url_suffix: + verbose_logger.debug( + "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix" + ) + + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="anthropic_text", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + return response + + +def _complete_anthropic(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.anthropic_key + or litellm.api_key + or os.environ.get("ANTHROPIC_API_KEY") + ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + # call /messages + # default route for all anthropic models + api_base = cast( + Optional[str], + api_base + or litellm.api_base + or get_secret("ANTHROPIC_API_BASE") + or get_secret("ANTHROPIC_BASE_URL") + or "https://api.anthropic.com/v1/messages", + ) + + # Check if we should disable automatic URL suffix appending + disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") + if ( + api_base is not None + and not disable_url_suffix + and not api_base.endswith("/v1/messages") + ): + api_base += "/v1/messages" + elif disable_url_suffix: + verbose_logger.debug( + "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix" + ) + + response = anthropic_chat_completions.completion( + model=model, + messages=messages, + api_base=api_base, + acompletion=acompletion, + custom_prompt_dict=litellm.custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), # for calculating input/output tokens + api_key=api_key, + logging_obj=logging, + headers=headers, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + ) + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + response = response + + return response + + +def _complete_nlp_cloud(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + nlp_cloud_key = ( + api_key + or litellm.nlp_cloud_key + or get_secret("NLP_CLOUD_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("NLP_CLOUD_API_BASE") + or "https://api.nlpcloud.io/v1/gpu/" + ) + + response = nlp_cloud_chat_completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=nlp_cloud_key, + logging_obj=logging, + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + response, + model, + custom_llm_provider="nlp_cloud", + logging_obj=logging, + ) + + if optional_params.get("stream", False) or acompletion is True: + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + ) + + response = response + + return response + + +def _complete_aleph_alpha(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + api_key = ctx.api_key + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + aleph_alpha_key = ( + api_key + or litellm.aleph_alpha_key + or get_secret("ALEPH_ALPHA_API_KEY") + or get_secret("ALEPHALPHA_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("ALEPH_ALPHA_API_BASE") + or "https://api.aleph-alpha.com/complete" + ) + + model_response = aleph_alpha.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + default_max_tokens_to_sample=litellm.max_tokens, + api_key=aleph_alpha_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="aleph_alpha", + logging_obj=logging, + ) + return response + response = model_response + + return response + + +def _complete_cohere_chat(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + extra_headers = ctx.extra_headers + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + cohere_key = ( + api_key + or litellm.cohere_key + or get_secret_str("COHERE_API_KEY") + or get_secret_str("CO_API_KEY") + or litellm.api_key + ) + + cohere_route = CohereModelInfo.get_cohere_route(model) + verbose_logger.debug(f"Cohere route: {cohere_route}") + # Set API base based on route + if cohere_route == "v2": + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.com/v2/chat" + ) + # Remove v2/ prefix from model name for the actual API call + if "v2/" in model: + model = model.replace("v2/", "") + else: + api_base = ( + api_base + or litellm.api_base + or get_secret_str("COHERE_API_BASE") + or "https://api.cohere.ai/v1/chat" + ) + + headers = headers or litellm.headers or {} + if headers is None: + headers = {} + + if extra_headers is not None: + headers.update(extra_headers) + + verbose_logger.debug(f"Model: {model}, API Base: {api_base}") + verbose_logger.debug(f"Provider Config: {provider_config}") + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cohere_chat", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=cohere_key, + provider_config=provider_config, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) + + return response + + +def _complete_maritalk(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + + maritalk_key = ( + api_key + or litellm.maritalk_key + or get_secret("MARITALK_API_KEY") + or litellm.api_key + ) + + api_base = ( + api_base + or litellm.api_base + or get_secret("MARITALK_API_BASE") + or "https://chat.maritaca.ai/api" + ) + + model_response = openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=maritalk_key, + logging_obj=logging, + custom_llm_provider="maritalk", + custom_prompt_dict=custom_prompt_dict, + ) + + response = model_response + + return response + + +def _complete_amazon_nova(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + api_key = ( + api_key + or litellm.amazon_nova_api_key + or get_secret_str("AMAZON_NOVA_API_KEY") + or litellm.api_key + ) + api_base = ( + api_base + or litellm.api_base + or get_secret_str("AMAZON_NOVA_API_BASE") + or "https://api.nova.amazon.com/v1" + ) + response = openai_like_chat_completion.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + ) + + return response + + +def _complete_huggingface(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + huggingface_key = ( + api_key + or litellm.huggingface_key + or os.environ.get("HF_TOKEN") + or os.environ.get("HUGGINGFACE_API_KEY") + or litellm.api_key + ) + hf_headers = headers or litellm.headers + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=hf_headers, + model_response=model_response, + api_key=huggingface_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + return response + + +def _complete_oci(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) + + return response + + +def _complete_compactifai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + stream = ctx.stream + timeout = ctx.timeout + + api_key = api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key + + api_base = api_base or "https://api.compactif.ai/v1" + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) - if custom_llm_provider == "azure": - # azure configs - ## check dynamic params ## - dynamic_params = False - if client is not None and ( - isinstance(client, openai.AzureOpenAI) - or isinstance(client, openai.AsyncAzureOpenAI) - ): - dynamic_params = _check_dynamic_azure_params( - azure_client_params={"api_version": api_version}, - azure_client=client, - ) + return response - api_type = get_secret("AZURE_API_TYPE") or "azure" - api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE") +def _complete_oobabooga(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params - api_version = ( - api_version - or litellm.api_version - or get_secret_str("AZURE_API_VERSION") - or litellm.AZURE_DEFAULT_API_VERSION - ) + model_response = oobabooga.completion( + model=model, + messages=messages, + model_response=model_response, + api_base=api_base, # type: ignore + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=None, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + ) + if "stream" in optional_params and optional_params["stream"] is True: + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="oobabooga", + logging_obj=logging, + ) + return response + response = model_response - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) + return response - azure_ad_token = optional_params.get("extra_body", {}).pop( - "azure_ad_token", None - ) or get_secret_str("AZURE_AD_TOKEN") - azure_ad_token_provider = litellm_params.get( - "azure_ad_token_provider", None - ) +def _complete_databricks(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base # for databricks we check in get_llm_provider and pass in the api base from there + or litellm.api_base + or os.getenv("DATABRICKS_API_BASE") + ) - headers = headers or litellm.headers + # set API KEY + api_key = ( + api_key + or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there + or litellm.databricks_key + or get_secret("DATABRICKS_API_KEY") + ) - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - if max_retries is not None: - optional_params["max_retries"] = max_retries + headers = headers or litellm.headers - if litellm.AzureOpenAIO1Config().is_o_series_model( - model=_azure_detection_model - ): - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIO1Config.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = azure_o1_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, # type: ignore - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - custom_llm_provider=custom_llm_provider, - ) - else: - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - ## COMPLETION CALL - response = azure_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=api_version, - api_type=api_type, - dynamic_params=dynamic_params, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, # type: ignore - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) + ## COMPLETION CALL + try: + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="databricks", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - elif custom_llm_provider == "azure_text": - # azure configs - api_type = get_secret_str("AZURE_API_TYPE") or "azure" + if optional_params.get("stream", False): + ## LOGGING + logging.post_call( + input=messages, + api_key=api_key, + original_response=response, + additional_args={"headers": headers}, + ) - api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") + return response - if api_base is None: - raise ValueError( - "api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable." - ) - api_version = ( - api_version - or litellm.api_version - or get_secret_str("AZURE_API_VERSION") - ) +def _complete_datarobot(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + stream = ctx.stream + timeout = ctx.timeout + + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=provider_config, + ) - api_key = ( - api_key - or litellm.api_key - or litellm.azure_key - or get_secret_str("AZURE_OPENAI_API_KEY") - or get_secret_str("AZURE_API_KEY") - ) + return response - azure_ad_token = optional_params.get("extra_body", {}).pop( - "azure_ad_token", None - ) or get_secret_str("AZURE_AD_TOKEN") - azure_ad_token_provider = litellm_params.get( - "azure_ad_token_provider", None - ) +def _complete_openrouter(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OPENROUTER_API_BASE") + or "https://openrouter.ai/api/v1" + ) - headers = headers or litellm.headers + api_key = ( + api_key + or litellm.api_key + or litellm.openrouter_key + or get_secret_str("OPENROUTER_API_KEY") + or get_secret_str("OR_API_KEY") + ) - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers + openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" + openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" - ## LOAD CONFIG - if set - config = litellm.AzureOpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > azure_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v + openrouter_headers = { + "HTTP-Referer": openrouter_site_url, + "X-Title": openrouter_app_name, + } - ## COMPLETION CALL - response = azure_text_completions.completion( - model=model, - messages=messages, - headers=headers, - api_key=api_key, - api_base=api_base, - api_version=cast(str, api_version), - api_type=api_type, - azure_ad_token=azure_ad_token, - azure_ad_token_provider=azure_ad_token_provider, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, # pass AsyncAzureOpenAI, AzureOpenAI client - ) + _headers = headers or litellm.headers + if _headers: + openrouter_headers.update(_headers) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={ - "headers": headers, - "api_version": api_version, - "api_base": api_base, - }, - ) - elif custom_llm_provider == "deepseek": - ## COMPLETION CALL + headers = openrouter_headers - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + ## Load Config + config = litellm.OpenrouterConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models + if "extra_body" in optional_params: + optional_params[k].update(v) + else: + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v - elif custom_llm_provider == "azure_ai": - from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="openrouter", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) + + return response - azure_ai_route = AzureFoundryModelInfo.get_azure_ai_route(model) - # Check if this is an agents route - model format: azure_ai/agents/ - if azure_ai_route == "agents": - from litellm.llms.azure_ai.agents import AzureAIAgentsConfig +def _complete_vercel_ai_gateway(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or litellm.api_base + or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") + or "https://ai-gateway.vercel.sh/v1" + ) - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure AI Agents requests require an api_base. " - "Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) + api_key = api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - response = AzureAIAgentsConfig.completion( - model=model, - messages=messages, - api_base=api_base, - api_key=api_key, - model_response=model_response, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - acompletion=acompletion, - stream=stream, - headers=headers or litellm.headers, - ) + vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" + vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM" - # Check if this is a Claude model - route to Azure Anthropic handler - elif "claude" in model.lower(): - # Use Azure Anthropic handler for Claude models - api_base = AzureFoundryModelInfo.get_api_base(api_base) - if api_base is None: - raise ValueError( - "Azure Anthropic requests require an api_base. " - "Set `api_base` or the AZURE_AI_API_BASE env var." - ) - api_key = AzureFoundryModelInfo.get_api_key(api_key) - - # Ensure the URL ends with /v1/messages for Anthropic - if api_base: - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/messages"): - if "/anthropic" in api_base: - parts = api_base.split("/anthropic", 1) - api_base = parts[0] + "/anthropic" - else: - api_base = api_base + "/anthropic" - api_base = api_base + "/v1/messages" + vercel_headers = { + "http-referer": vercel_site_url, + "x-title": vercel_app_name, + } - response = azure_anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response + _headers = headers or litellm.headers + if _headers: + vercel_headers.update(_headers) + + headers = vercel_headers + + ## Load Config + config = litellm.VercelAIGatewayConfig.get_config() + for k, v in config.items(): + if k == "extra_body": + # we use openai 'extra_body' to pass vercel specific params - providerOptions + if "extra_body" in optional_params: + optional_params[k].update(v) else: - # Non-Claude models use standard Azure AI flow - api_base = AzureFoundryModelInfo.get_api_base(api_base) - # set API KEY - api_key = AzureFoundryModelInfo.get_api_key(api_key) + optional_params[k] = v + elif k not in optional_params: + optional_params[k] = v + + ## COMPLETION CALL + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="vercel_ai_gateway", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + ## LOGGING + logging.post_call( + input=messages, api_key=openai.api_key, original_response=response + ) - headers = headers or litellm.headers + return response - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - ## FOR COHERE - if "command-r" in model: # make sure tool call in messages are str - messages = stringify_json_tool_call_content(messages=messages) +def _complete_vertex_ai_beta(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, # pass AsyncOpenAI, OpenAI client - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) - elif ( - custom_llm_provider == "text-completion-openai" - or "ft:babbage-002" in model - or "ft:davinci-002" in model # support for finetuned completion models - or custom_llm_provider - in litellm.openai_text_completion_compatible_providers - and kwargs.get("text_completion") is True - ): - openai.api_type = "openai" + gemini_api_key = ( + api_key + or get_api_key_from_env() + or get_secret("PALM_API_KEY") # older palm api key should also work + or litellm.api_key + ) - api_base = ( - api_base - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) + api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") + new_params = safe_deep_copy(optional_params or {}) + response = vertex_chat_completion.completion( # type: ignore + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=gemini_api_key, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, # type: ignore + client=client, + api_base=api_base, + extra_headers=headers, + ) - openai.api_version = None - # set API KEY + return response - api_key = ( - api_key - or litellm.api_key - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) - headers = headers or litellm.headers +def _complete_vertex_ai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret("VERTEXAI_PROJECT") + ) + vertex_ai_location = ( + optional_params.pop("vertex_location", None) + or optional_params.pop("vertex_ai_location", None) + or litellm.vertex_location + or get_secret("VERTEXAI_LOCATION") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret("VERTEXAI_CREDENTIALS") + ) + + api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") + + new_params = safe_deep_copy(optional_params or {}) + model_route = get_vertex_ai_model_route(model=model, litellm_params=litellm_params) + + if model_route == VertexAIModelRoute.PARTNER_MODELS: + model_response = vertex_partner_models_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.GEMINI: + model_response = vertex_chat_completion.completion( # type: ignore + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + gemini_api_key=None, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + custom_llm_provider=custom_llm_provider, # type: ignore + client=client, + api_base=api_base, + extra_headers=headers, + ) + elif model_route == VertexAIModelRoute.GEMMA: + # Vertex Gemma Models with custom prediction endpoint + model_response = vertex_gemma_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.MODEL_GARDEN: + # Vertex Model Garden - OpenAI compatible models + model_response = vertex_model_garden_chat_completion.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + api_base=api_base, + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + headers=headers, + custom_prompt_dict=custom_prompt_dict, + timeout=timeout, + client=client, + ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config = VertexAgentEngineConfig() - ## LOAD CONFIG - if set - config = litellm.OpenAITextCompletionConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_text_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - if litellm.organization: - openai.organization = litellm.organization + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials - if ( - len(messages) > 0 - and "content" in messages[0] - and isinstance(messages[0]["content"], list) - ): - # text-davinci-003 can accept a string or array, if it's an array, assume the array is set in messages[0]['content'] - # https://platform.openai.com/docs/api-reference/completions/create - prompt = messages[0]["content"] - else: - prompt = " ".join([message["content"] for message in messages]) # type: ignore + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + encoding=_get_encoding(), + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) + else: # VertexAIModelRoute.NON_GEMINI + model_response = vertex_ai_non_gemini.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=new_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + vertex_location=vertex_ai_location, + vertex_project=vertex_ai_project, + vertex_credentials=vertex_credentials, + logging_obj=logging, + acompletion=acompletion, + ) - ## COMPLETION CALL - _response = openai_text_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - acompletion=acompletion, - client=client, # pass AsyncOpenAI, OpenAI client + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vertex_ai", logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore ) + return response + response = model_response - if ( - optional_params.get("stream", False) is False - and acompletion is False - and text_completion is False - ): - # convert to chat completion response - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) + return response - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - response = _response - elif custom_llm_provider == "fireworks_ai": - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - elif custom_llm_provider == "heroku": - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - elif custom_llm_provider == "ragflow": - ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - elif custom_llm_provider == "xai": - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - elif custom_llm_provider == "groq": - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("GROQ_API_BASE") - or "https://api.groq.com/openai/v1" - ) +def _complete_predibase(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + tenant_id = ( + optional_params.pop("tenant_id", None) + or optional_params.pop("predibase_tenant_id", None) + or litellm.predibase_tenant_id + or get_secret("PREDIBASE_TENANT_ID") + ) - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.groq_key - or get_secret("GROQ_API_KEY") - ) + if tenant_id is None: + raise ValueError( + "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." + ) - headers = headers or litellm.headers + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or get_secret("PREDIBASE_API_BASE") + ) - ## LOAD CONFIG - if set - config = litellm.GroqChatConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v + api_key = ( + api_key + or litellm.api_key + or litellm.predibase_key + or get_secret("PREDIBASE_API_KEY") + ) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - elif custom_llm_provider == "bedrock_mantle": - api_base = ( - api_base or litellm.api_base or get_secret("BEDROCK_MANTLE_API_BASE") - ) - api_key = api_key or litellm.api_key or get_secret("BEDROCK_MANTLE_API_KEY") - headers = headers or litellm.headers - config = litellm.BedrockMantleChatConfig.get_config() - for k, v in config.items(): - if k not in optional_params: - optional_params[k] = v - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - elif custom_llm_provider == "a2a": - # A2A (Agent-to-Agent) Protocol - # Resolve agent configuration from registry if model format is "a2a/" - ( - api_base, - api_key, - headers, - ) = litellm.A2AConfig.resolve_agent_config_from_registry( - model=model, - api_base=api_base, - api_key=api_key, - headers=headers, - optional_params=optional_params, - ) + _model_response = predibase_chat_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + tenant_id=tenant_id, + timeout=timeout, + ) - # Fall back to environment variables and defaults - api_base = api_base or litellm.api_base or get_secret_str("A2A_API_BASE") + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + return _model_response + response = _model_response - if api_base is None: - raise Exception( - "api_base is required for A2A provider. " - "Either provide api_base parameter, set A2A_API_BASE environment variable, " - "or register the agent in the proxy with model='a2a/'." - ) + return response - headers = headers or litellm.headers - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - elif custom_llm_provider == "gigachat": - # GigaChat - Sber AI's LLM (Russia) - api_key = ( - api_key - or litellm.api_key - or litellm.gigachat_key - or get_secret("GIGACHAT_API_KEY") - or get_secret("GIGACHAT_CREDENTIALS") - ) +def _complete_text_completion_codestral(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + or litellm.api_base + or "https://codestral.mistral.ai/v1/fim/completions" + ) - headers = headers or litellm.headers or {} + api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + text_completion_model_response = litellm.TextCompletionResponse(stream=stream) + + _model_response = codestral_text_completions.completion( # type: ignore + model=model, + messages=messages, + model_response=text_completion_model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + api_base=api_base, + custom_prompt_dict=custom_prompt_dict, + api_key=api_key, + timeout=timeout, + ) + + if ( + "stream" in optional_params + and optional_params["stream"] is True + and acompletion is False + ): + return _model_response + response = _model_response - elif custom_llm_provider == "sap": - headers = headers or litellm.headers - ## LOAD CONFIG - if set - config = litellm.GenAIHubOrchestrationConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v - - response = sap_gen_ai_hub_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - shared_session=shared_session, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - api_key=api_key, - api_base=api_base, - stream=stream, - ) - elif custom_llm_provider == "aiohttp_openai": - # NEW aiohttp provider for 10-100x higher RPS - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) + return response - headers = headers or litellm.headers - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers - response = base_llm_aiohttp_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - elif custom_llm_provider == "cometapi": - api_key = ( - api_key - or litellm.cometapi_key - or get_secret_str("COMETAPI_KEY") - or litellm.api_key - ) +def _complete_text_completion_inception(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + text_completion = ctx.text_completion + timeout = ctx.timeout + + passed_api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + ) + api_base = ( + passed_api_base + or get_secret_str("INCEPTION_API_BASE") + or "https://api.inceptionlabs.ai/v1" + ) + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = ( + api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + ) - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COMETAPI_API_BASE") - or "https://api.cometapi.com/v1" - ) + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, # type: ignore[arg-type] + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + _response = ( + litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response ) + ) - ## LOGGING - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) - elif custom_llm_provider == "minimax": - api_key = api_key or get_secret_str("MINIMAX_API_KEY") or litellm.api_key + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response - api_base = ( - api_base - or litellm.api_base - or get_secret_str("MINIMAX_API_BASE") - or "https://api.minimax.io/v1" - ) + return response - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) - elif custom_llm_provider == "hosted_vllm": - api_base = ( - api_base or litellm.api_base or get_secret_str("HOSTED_VLLM_API_BASE") - ) - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - logging.post_call( - input=messages, api_key=api_key, original_response=response - ) - elif ( - model in litellm.open_ai_chat_completion_models - or custom_llm_provider == "custom_openai" - or custom_llm_provider == "deepinfra" - or custom_llm_provider == "perplexity" - or custom_llm_provider == "nvidia_nim" - or custom_llm_provider == "cerebras" - or custom_llm_provider == "baseten" - or custom_llm_provider == "sambanova" - or custom_llm_provider == "volcengine" - or custom_llm_provider == "anyscale" - or custom_llm_provider == "openai" - or custom_llm_provider == "together_ai" - or custom_llm_provider == "nebius" - or custom_llm_provider == "wandb" - or custom_llm_provider == "clarifai" - or custom_llm_provider in litellm.openai_compatible_providers - or JSONProviderRegistry.exists( - custom_llm_provider - ) # JSON-configured providers - or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo - ): # allow user to make an openai call with a custom base - # note: if a user sets a custom base - we should ensure this works - # allow for the setting of dynamic and stateful api-bases - api_base = ( - api_base # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or get_secret("OPENAI_BASE_URL") - or get_secret("OPENAI_API_BASE") - or "https://api.openai.com/v1" - ) - organization = ( - organization - or litellm.organization - or get_secret("OPENAI_ORGANIZATION") - or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 - ) - openai.organization = organization - # set API KEY - api_key = ( - api_key - or litellm.api_key # for deepinfra/perplexity/anyscale/friendliai we check in get_llm_provider and pass in the api key from there - or litellm.openai_key - or get_secret("OPENAI_API_KEY") - ) +def _complete_sagemaker_chat(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) - headers = headers or litellm.headers + ## RESPONSE OBJECT + response = model_response - # Add GitHub Copilot headers (same as /responses endpoint does) - if custom_llm_provider == "github_copilot": - from litellm.llms.github_copilot.authenticator import Authenticator - from litellm.llms.github_copilot.common_utils import ( - get_copilot_default_headers, - ) + return response - copilot_auth = Authenticator() - copilot_api_key = copilot_auth.get_api_key() - copilot_headers = get_copilot_default_headers(copilot_api_key) - if extra_headers: - copilot_headers.update(extra_headers) - extra_headers = copilot_headers - if extra_headers is not None: - optional_params["extra_headers"] = extra_headers +def _complete_sagemaker(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + custom_prompt_dict = ctx.custom_prompt_dict + hf_model_name = ctx.hf_model_name + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params - if ( - litellm.enable_preview_features and metadata is not None - ): # [PREVIEW] allow metadata to be passed to OPENAI - openai_metadata = get_requester_metadata(metadata) - if openai_metadata is not None: - optional_params["metadata"] = openai_metadata - - ## LOAD CONFIG - if set - config = litellm.OpenAIConfig.get_config() - for k, v in config.items(): - if ( - k not in optional_params - ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in - optional_params[k] = v + model_response = sagemaker_llm.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + custom_prompt_dict=custom_prompt_dict, + hf_model_name=hf_model_name, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + acompletion=acompletion, + ) + + ## RESPONSE OBJECT + response = model_response + + return response + + +def _complete_bedrock(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + provider_config = ctx.provider_config + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + + if "aws_bedrock_client" in optional_params: + verbose_logger.warning( + "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." + ) + # Extract credentials for legacy boto3 client and pass thru to httpx + aws_bedrock_client = optional_params.pop("aws_bedrock_client") + creds = aws_bedrock_client._get_credentials().get_frozen_credentials() + + if creds.access_key: + optional_params["aws_access_key_id"] = creds.access_key + if creds.secret_key: + optional_params["aws_secret_access_key"] = creds.secret_key + if creds.token: + optional_params["aws_session_token"] = creds.token + if ( + "aws_region_name" not in optional_params + or optional_params["aws_region_name"] is None + ): + optional_params["aws_region_name"] = aws_bedrock_client.meta.region_name + + bedrock_route = BedrockModelInfo.get_bedrock_route(model) + if bedrock_route == "claude_platform": + provider_config = ProviderConfigManager.get_provider_chat_config( + model=model, + provider=LlmProviders.BEDROCK, + ) + model = BedrockModelInfo.get_claude_platform_model(model) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + provider_config=provider_config, + ) + return response + elif bedrock_route == "converse": + model = model.replace("converse/", "") + response = bedrock_converse_chat_completion.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, # type: ignore + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + extra_headers=headers, # Use merged headers instead of original extra_headers + timeout=timeout, + acompletion=acompletion, + client=client, + api_base=api_base, + api_key=api_key, + ) + elif bedrock_route == "converse_like": + model = model.replace("converse_like/", "") + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) + else: + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + custom_llm_provider="bedrock", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) - ## COMPLETION CALL - use_base_llm_http_handler = get_secret_bool( - "EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER" - ) + return response - try: - if use_base_llm_http_handler: - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - else: - response = openai_chat_completions.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - organization=organization, - custom_llm_provider=custom_llm_provider, - shared_session=shared_session, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) +def _complete_watsonx(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + timeout = ctx.timeout + + response = watsonx_chat_completion.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=_get_encoding(), + custom_llm_provider="watsonx", + ) - elif custom_llm_provider == "mistral": - api_key = api_key or litellm.api_key or get_secret("MISTRAL_API_KEY") - api_base = ( - api_base - or litellm.api_base - or get_secret("MISTRAL_API_BASE") - or "https://api.mistral.ai/v1" - ) + return response - response = base_llm_http_handler.completion( - model=model, - messages=messages, - api_base=api_base, - custom_llm_provider=custom_llm_provider, - model_response=model_response, - encoding=_get_encoding(), - logging_obj=logging, - optional_params=optional_params, - timeout=timeout, - litellm_params=litellm_params, - shared_session=shared_session, - acompletion=acompletion, - stream=stream, - api_key=api_key, - headers=headers, - client=client, - provider_config=provider_config, - ) - elif ( - "replicate" in model - or custom_llm_provider == "replicate" - or model in litellm.replicate_models - ): - # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") - replicate_key = ( - api_key - or litellm.replicate_key - or litellm.api_key - or get_secret("REPLICATE_API_KEY") - or get_secret("REPLICATE_API_TOKEN") - ) - api_base = ( - api_base - or litellm.api_base - or get_secret("REPLICATE_API_BASE") - or "https://api.replicate.com/v1" - ) +def _complete_watsonx_text(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + api_key = ( + api_key + or optional_params.pop("apikey", None) + or get_secret_str("WATSONX_APIKEY") + or get_secret_str("WATSONX_API_KEY") + or get_secret_str("WX_API_KEY") + ) - model_response = replicate_chat_completion( # type: ignore - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=replicate_key, - logging_obj=logging, - custom_prompt_dict=custom_prompt_dict, - acompletion=acompletion, - headers=headers, - ) + api_base = ( + api_base + or optional_params.pop( + "url", + optional_params.pop("api_base", optional_params.pop("base_url", None)), + ) + or get_secret_str("WATSONX_API_BASE") + or get_secret_str("WATSONX_URL") + or get_secret_str("WX_URL") + or get_secret_str("WML_URL") + ) - if optional_params.get("stream", False) is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=replicate_key, - original_response=model_response, - ) + wx_credentials = optional_params.pop( + "wx_credentials", + optional_params.pop( + "watsonx_credentials", None + ), # follow {provider}_credentials, same as vertex ai + ) - response = model_response - elif ( - "clarifai" in model - or custom_llm_provider == "clarifai" - or model in litellm.clarifai_models - ): - pass # Deprecated - handled in the openai compatible provider section above - elif custom_llm_provider == "anthropic_text": - api_key = ( - api_key - or litellm.anthropic_key - or litellm.api_key - or os.environ.get("ANTHROPIC_API_KEY") - ) - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - api_base = ( - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/complete" - ) + token: Optional[str] = None + if wx_credentials is not None: + api_base = wx_credentials.get("url", api_base) + api_key = wx_credentials.get("apikey", wx_credentials.get("api_key", api_key)) + token = wx_credentials.get( + "token", + wx_credentials.get( + "watsonx_token", None + ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' + ) - # Check if we should disable automatic URL suffix appending - disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if ( - api_base is not None - and not disable_url_suffix - and not api_base.endswith("/v1/complete") - ): - api_base += "/v1/complete" - elif disable_url_suffix: - verbose_logger.debug( - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix" - ) + if token is not None: + optional_params["token"] = token - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="anthropic_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - elif custom_llm_provider == "anthropic": - api_key = ( - api_key - or litellm.anthropic_key - or litellm.api_key - or os.environ.get("ANTHROPIC_API_KEY") - ) - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - # call /messages - # default route for all anthropic models - api_base = ( - api_base - or litellm.api_base - or get_secret("ANTHROPIC_API_BASE") - or get_secret("ANTHROPIC_BASE_URL") - or "https://api.anthropic.com/v1/messages" - ) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="watsonx_text", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) - # Check if we should disable automatic URL suffix appending - disable_url_suffix = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if ( - api_base is not None - and not disable_url_suffix - and not api_base.endswith("/v1/messages") - ): - api_base += "/v1/messages" - elif disable_url_suffix: - verbose_logger.debug( - "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix" - ) + return response - response = anthropic_chat_completions.completion( - model=model, - messages=messages, - api_base=api_base, - acompletion=acompletion, - custom_prompt_dict=litellm.custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), # for calculating input/output tokens - api_key=api_key, - logging_obj=logging, - headers=headers, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - ) - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) - response = response - elif custom_llm_provider == "nlp_cloud": - nlp_cloud_key = ( - api_key - or litellm.nlp_cloud_key - or get_secret("NLP_CLOUD_API_KEY") - or litellm.api_key - ) - api_base = ( - api_base - or litellm.api_base - or get_secret("NLP_CLOUD_API_BASE") - or "https://api.nlpcloud.io/v1/gpu/" - ) +def _complete_vllm(ctx: _CompletionDispatchContext): + custom_prompt_dict = ctx.custom_prompt_dict + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params - response = nlp_cloud_chat_completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=nlp_cloud_key, - logging_obj=logging, - ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + model_response = vllm_handler.completion( + model=model, + messages=messages, + custom_prompt_dict=custom_prompt_dict, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + ) - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - response, - model, - custom_llm_provider="nlp_cloud", - logging_obj=logging, - ) + if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] + # don't try to access stream object, + response = CustomStreamWrapper( + model_response, + model, + custom_llm_provider="vllm", + logging_obj=logging, + ) + return response - if optional_params.get("stream", False) or acompletion is True: - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - ) + ## RESPONSE OBJECT + response = model_response - response = response - elif custom_llm_provider == "aleph_alpha": - aleph_alpha_key = ( - api_key - or litellm.aleph_alpha_key - or get_secret("ALEPH_ALPHA_API_KEY") - or get_secret("ALEPHALPHA_API_KEY") - or litellm.api_key - ) + return response - api_base = ( - api_base - or litellm.api_base - or get_secret("ALEPH_ALPHA_API_BASE") - or "https://api.aleph-alpha.com/complete" - ) - model_response = aleph_alpha.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - default_max_tokens_to_sample=litellm.max_tokens, - api_key=aleph_alpha_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) +def _complete_ollama(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="aleph_alpha", - logging_obj=logging, - ) - return response - response = model_response - elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": - cohere_key = ( - api_key - or litellm.cohere_key - or get_secret_str("COHERE_API_KEY") - or get_secret_str("CO_API_KEY") - or litellm.api_key - ) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) - cohere_route = CohereModelInfo.get_cohere_route(model) - verbose_logger.debug(f"Cohere route: {cohere_route}") - # Set API base based on route - if cohere_route == "v2": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COHERE_API_BASE") - or "https://api.cohere.com/v2/chat" - ) - # Remove v2/ prefix from model name for the actual API call - if "v2/" in model: - model = model.replace("v2/", "") - else: - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COHERE_API_BASE") - or "https://api.cohere.ai/v1/chat" - ) + return response - headers = headers or litellm.headers or {} - if headers is None: - headers = {} - if extra_headers is not None: - headers.update(extra_headers) +def _complete_ollama_chat(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = ( + litellm.api_base + or api_base + or get_secret("OLLAMA_API_BASE") + or "http://localhost:11434" + ) - verbose_logger.debug(f"Model: {model}, API Base: {api_base}") - verbose_logger.debug(f"Provider Config: {provider_config}") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cohere_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=cohere_key, - provider_config=provider_config, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) - elif custom_llm_provider == "maritalk": - maritalk_key = ( - api_key - or litellm.maritalk_key - or get_secret("MARITALK_API_KEY") - or litellm.api_key - ) + api_key = ( + api_key + or litellm.ollama_key + or os.environ.get("OLLAMA_API_KEY") + or litellm.api_key + ) + if api_key is not None and "Authorization" not in headers: + headers["Authorization"] = f"Bearer {api_key}" - api_base = ( - api_base - or litellm.api_base - or get_secret("MARITALK_API_BASE") - or "https://chat.maritaca.ai/api" - ) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="ollama_chat", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + client=client, + ) - model_response = openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=maritalk_key, - logging_obj=logging, - custom_llm_provider="maritalk", - custom_prompt_dict=custom_prompt_dict, - ) + return response - response = model_response - elif custom_llm_provider == "amazon_nova": - api_key = ( - api_key - or litellm.amazon_nova_api_key - or get_secret_str("AMAZON_NOVA_API_KEY") - or litellm.api_key - ) - api_base = ( - api_base - or litellm.api_base - or get_secret_str("AMAZON_NOVA_API_BASE") - or "https://api.nova.amazon.com/v1" - ) - response = openai_like_chat_completion.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - timeout=timeout, - custom_llm_provider=custom_llm_provider, - custom_prompt_dict=custom_prompt_dict, - ) - elif custom_llm_provider == "huggingface": - huggingface_key = ( - api_key - or litellm.huggingface_key - or os.environ.get("HF_TOKEN") - or os.environ.get("HUGGINGFACE_API_KEY") - or litellm.api_key - ) - hf_headers = headers or litellm.headers - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=hf_headers, - model_response=model_response, - api_key=huggingface_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - elif custom_llm_provider == "oci": - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - ) - elif custom_llm_provider == "compactifai": - api_key = ( - api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key - ) - api_base = api_base or "https://api.compactif.ai/v1" +def _complete_triton(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = litellm.api_base or api_base + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + ) - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - elif custom_llm_provider == "oobabooga": - custom_llm_provider = "oobabooga" - model_response = oobabooga.completion( - model=model, - messages=messages, - model_response=model_response, - api_base=api_base, # type: ignore - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - api_key=None, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) - if "stream" in optional_params and optional_params["stream"] is True: - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="oobabooga", - logging_obj=logging, - ) - return response - response = model_response - elif custom_llm_provider == "databricks": - api_base = ( - api_base # for databricks we check in get_llm_provider and pass in the api base from there - or litellm.api_base - or os.getenv("DATABRICKS_API_BASE") - ) + return response - # set API KEY - api_key = ( - api_key - or litellm.api_key # for databricks we check in get_llm_provider and pass in the api key from there - or litellm.databricks_key - or get_secret("DATABRICKS_API_KEY") - ) - headers = headers or litellm.headers +def _complete_cloudflare(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout - ## COMPLETION CALL - try: - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="databricks", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e + api_key = ( + api_key + or litellm.cloudflare_api_key + or litellm.api_key + or get_secret("CLOUDFLARE_API_KEY") + ) + account_id = get_secret("CLOUDFLARE_ACCOUNT_ID") + api_base = ( + api_base + or litellm.api_base + or get_secret("CLOUDFLARE_API_BASE") + or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" + ) - if optional_params.get("stream", False): - ## LOGGING - logging.post_call( - input=messages, - api_key=api_key, - original_response=response, - additional_args={"headers": headers}, - ) + custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="cloudflare", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements + ) - elif custom_llm_provider == "datarobot": - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=provider_config, - ) - elif custom_llm_provider == "openrouter": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OPENROUTER_API_BASE") - or "https://openrouter.ai/api/v1" - ) + return response - api_key = ( - api_key - or litellm.api_key - or litellm.openrouter_key - or get_secret_str("OPENROUTER_API_KEY") - or get_secret_str("OR_API_KEY") - ) - openrouter_site_url = get_secret("OR_SITE_URL") or "https://litellm.ai" - openrouter_app_name = get_secret("OR_APP_NAME") or "liteLLM" +def _complete_petals(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + client = ctx.client + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream - openrouter_headers = { - "HTTP-Referer": openrouter_site_url, - "X-Title": openrouter_app_name, - } + api_base = api_base or litellm.api_base - _headers = headers or litellm.headers - if _headers: - openrouter_headers.update(_headers) + stream = optional_params.pop("stream", False) + model_response = petals_handler.completion( + model=model, + messages=messages, + api_base=api_base, + model_response=model_response, + print_verbose=print_verbose, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + encoding=_get_encoding(), + logging_obj=logging, + client=client, + ) + if stream is True: ## [BETA] + # Fake streaming for petals + resp_string = model_response["choices"][0]["message"]["content"] + response = CustomStreamWrapper( + resp_string, + model, + custom_llm_provider="petals", + logging_obj=logging, + ) + return response + response = model_response - headers = openrouter_headers + return response - ## Load Config - config = litellm.OpenrouterConfig.get_config() - for k, v in config.items(): - if k == "extra_body": - # we use openai 'extra_body' to pass openrouter specific params - transforms, route, models - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - data = {"model": model, "messages": messages, **optional_params} +def _complete_snowflake(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="openrouter", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call( - input=messages, api_key=openai.api_key, original_response=response - ) - elif custom_llm_provider == "vercel_ai_gateway": - api_base = ( - api_base - or litellm.api_base - or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") - or "https://ai-gateway.vercel.sh/v1" - ) + try: + client = ( + HTTPHandler(timeout=timeout) if stream is False else None + ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + ) - api_key = ( - api_key or litellm.api_key or get_secret("VERCEL_AI_GATEWAY_API_KEY") - ) + except Exception as e: + ## LOGGING - log the original exception returned + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e - vercel_site_url = get_secret("VERCEL_SITE_URL") or "https://litellm.ai" - vercel_app_name = get_secret("VERCEL_APP_NAME") or "liteLLM" + return response - vercel_headers = { - "http-referer": vercel_site_url, - "x-title": vercel_app_name, - } - _headers = headers or litellm.headers - if _headers: - vercel_headers.update(_headers) +def _complete_gradient_ai(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + api_base = litellm.api_base or api_base + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider="gradient_ai", + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + ) - headers = vercel_headers + return response - ## Load Config - config = litellm.VercelAIGatewayConfig.get_config() - for k, v in config.items(): - if k == "extra_body": - # we use openai 'extra_body' to pass vercel specific params - providerOptions - if "extra_body" in optional_params: - optional_params[k].update(v) - else: - optional_params[k] = v - elif k not in optional_params: - optional_params[k] = v - data = {"model": model, "messages": messages, **optional_params} +def _complete_bytez(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout - ## COMPLETION CALL - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="vercel_ai_gateway", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - ## LOGGING - logging.post_call( - input=messages, api_key=openai.api_key, original_response=response - ) - elif ( - custom_llm_provider == "together_ai" - or ("togethercomputer" in model) - or (model in litellm.together_ai_models) - ): - """ - Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility - """ - pass - elif custom_llm_provider == "palm": - raise ValueError( - "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" - ) - elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) + api_key = ( + api_key + or litellm.bytez_key + or get_secret_str("BYTEZ_API_KEY") + or litellm.api_key + ) - gemini_api_key = ( - api_key - or get_api_key_from_env() - or get_secret("PALM_API_KEY") # older palm api key should also work - or litellm.api_key - ) + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=bytez_transformation, + ) - api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") - new_params = safe_deep_copy(optional_params or {}) - response = vertex_chat_completion.completion( # type: ignore - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=gemini_api_key, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, # type: ignore - client=client, - api_base=api_base, - extra_headers=headers, - ) + pass - elif custom_llm_provider == "vertex_ai": - vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) - or litellm.vertex_project - or get_secret("VERTEXAI_PROJECT") - ) - vertex_ai_location = ( - optional_params.pop("vertex_location", None) - or optional_params.pop("vertex_ai_location", None) - or litellm.vertex_location - or get_secret("VERTEXAI_LOCATION") - ) - vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) - or get_secret("VERTEXAI_CREDENTIALS") - ) + return response - api_base = api_base or litellm.api_base or get_secret("VERTEXAI_API_BASE") - new_params = safe_deep_copy(optional_params or {}) - model_route = get_vertex_ai_model_route( - model=model, litellm_params=litellm_params - ) +def _complete_lemonade(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout - if model_route == VertexAIModelRoute.PARTNER_MODELS: - model_response = vertex_partner_models_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.GEMINI: - model_response = vertex_chat_completion.completion( # type: ignore - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - gemini_api_key=None, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - custom_llm_provider=custom_llm_provider, # type: ignore - client=client, - api_base=api_base, - extra_headers=headers, - ) - elif model_route == VertexAIModelRoute.GEMMA: - # Vertex Gemma Models with custom prediction endpoint - model_response = vertex_gemma_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.MODEL_GARDEN: - # Vertex Model Garden - OpenAI compatible models - model_response = vertex_model_garden_chat_completion.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - api_base=api_base, - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - headers=headers, - custom_prompt_dict=custom_prompt_dict, - timeout=timeout, - client=client, - ) - elif model_route == VertexAIModelRoute.AGENT_ENGINE: - # Vertex AI Agent Engine (Reasoning Engines) - from litellm.llms.vertex_ai.agent_engine.transformation import ( - VertexAgentEngineConfig, - ) + api_key = ( + api_key + or litellm.lemonade_key + or get_secret_str("LEMONADE_API_KEY") + or litellm.api_key + ) - vertex_agent_engine_config = VertexAgentEngineConfig() + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=lemonade_transformation, + ) - # Update litellm_params with vertex credentials - litellm_params["vertex_project"] = vertex_ai_project - litellm_params["vertex_location"] = vertex_ai_location - litellm_params["vertex_credentials"] = vertex_credentials + pass - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - model_response=model_response, - optional_params=new_params, - litellm_params=litellm_params, # type: ignore - encoding=_get_encoding(), - api_key=None, - api_base=api_base, - logging_obj=logging, - acompletion=acompletion, - timeout=timeout, - client=client, - custom_llm_provider="vertex_ai", - provider_config=vertex_agent_engine_config, - headers=headers or {}, - ) - else: # VertexAIModelRoute.NON_GEMINI - model_response = vertex_ai_non_gemini.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=new_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - vertex_location=vertex_ai_location, - vertex_project=vertex_ai_project, - vertex_credentials=vertex_credentials, - logging_obj=logging, - acompletion=acompletion, - ) + return response - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vertex_ai", - logging_obj=logging, - ) - return response - response = model_response - elif custom_llm_provider == "predibase": - tenant_id = ( - optional_params.pop("tenant_id", None) - or optional_params.pop("predibase_tenant_id", None) - or litellm.predibase_tenant_id - or get_secret("PREDIBASE_TENANT_ID") - ) - if tenant_id is None: - raise ValueError( - "Missing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=)`) or in env - `PREDIBASE_TENANT_ID`." - ) +def _complete_ovhcloud(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or get_secret("PREDIBASE_API_BASE") - ) + api_key = ( + api_key + or litellm.ovhcloud_key + or get_secret_str("OVHCLOUD_API_KEY") + or litellm.api_key + ) - api_key = ( - api_key - or litellm.api_key - or litellm.predibase_key - or get_secret("PREDIBASE_API_KEY") - ) + api_base = ( + api_base + or litellm.api_base + or get_secret_str("OVHCLOUD_API_BASE") + or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" + ) - _model_response = predibase_chat_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - tenant_id=tenant_id, - timeout=timeout, - ) + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + timeout=timeout, # type: ignore + client=client, + custom_llm_provider=custom_llm_provider, + encoding=_get_encoding(), + stream=stream, + provider_config=ovhcloud_transformation, + ) + + pass + + return response + + +def _complete_custom(ctx: _CompletionDispatchContext): + api_base = ctx.api_base + headers = ctx.headers + kwargs = ctx.kwargs + max_tokens = ctx.max_tokens + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + temperature = ctx.temperature + top_p = ctx.top_p + + url = litellm.api_base or api_base or "" + if url is None or url == "": + raise ValueError( + "api_base not set. Set api_base or litellm.api_base for custom endpoints" + ) + + """ + assume input to custom LLM api bases follow this format: + resp = litellm.module_level_client.post( + api_base, + json={ + 'model': 'meta-llama/Llama-2-13b-hf', # model name + 'params': { + 'prompt': ["The capital of France is P"], + 'max_tokens': 32, + 'temperature': 0.7, + 'top_p': 1.0, + 'top_k': 40, + } + } + ) + + """ + prompt = " ".join([message["content"] for message in messages]) # type: ignore + resp = litellm.module_level_client.post( + url, + headers=headers, + json={ + "model": model, + "params": { + "prompt": [prompt], + "max_tokens": max_tokens, + "temperature": temperature, + "top_p": top_p, + "top_k": kwargs.get("top_k"), + }, + **kwargs.get("extra_body", {}), + }, + ) + response_json = resp.json() + """ + assume all responses from custom api_bases of this format: + { + 'data': [ + { + 'prompt': 'The capital of France is P', + 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], + 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], + 'message': 'ok' + } + ] + } + """ + string_response = response_json["data"][0]["output"][0] + ## RESPONSE OBJECT + model_response.choices[0].message.content = string_response # type: ignore + model_response.created = int(time.time()) + model_response.model = model + response = model_response - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - return _model_response - response = _model_response - elif custom_llm_provider == "text-completion-codestral": - api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - or litellm.api_base - or "https://codestral.mistral.ai/v1/fim/completions" - ) + return response - api_key = api_key or litellm.api_key or get_secret("CODESTRAL_API_KEY") - text_completion_model_response = litellm.TextCompletionResponse( - stream=stream - ) +def _complete_custom_providers(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + custom_prompt_dict = ctx.custom_prompt_dict + headers = ctx.headers + litellm_params = ctx.litellm_params + logger_fn = ctx.logger_fn + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + stream = ctx.stream + timeout = ctx.timeout + + custom_handler: Optional[CustomLLM] = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider( + model=model, custom_llm_provider=custom_llm_provider + ) - _model_response = codestral_text_completions.completion( # type: ignore - model=model, - messages=messages, - model_response=text_completion_model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - api_base=api_base, - custom_prompt_dict=custom_prompt_dict, - api_key=api_key, - timeout=timeout, - ) + ## ROUTE LLM CALL ## + handler_fn = custom_chat_llm_router( + async_fn=acompletion, stream=stream, custom_llm=custom_handler + ) - if ( - "stream" in optional_params - and optional_params["stream"] is True - and acompletion is False - ): - return _model_response - response = _model_response - elif custom_llm_provider == "text-completion-inception": - passed_api_base = ( - api_base - or optional_params.pop("api_base", None) - or optional_params.pop("base_url", None) - ) - api_base = ( - passed_api_base - or get_secret_str("INCEPTION_API_BASE") - or "https://api.inceptionlabs.ai/v1" - ) - # FIM is served at `/v1/fim/completions`; the OpenAI client appends - # `/completions`, so point it at the `/v1/fim` base. - api_base = api_base.rstrip("/") - if not api_base.endswith("/fim"): - api_base += "/fim" + headers = headers or litellm.headers or {} - # Don't forward the server-managed Inception key to a caller-supplied - # api_base; only resolve it for the default/server base, or when the - # caller passes their own key. - if passed_api_base is None or api_key: - api_key = ( - api_key - or litellm.inception_key - or get_secret_str("INCEPTION_API_KEY") - ) + ## CALL FUNCTION + response = handler_fn( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + custom_prompt_dict=custom_prompt_dict, + client=client, # pass AsyncOpenAI, OpenAI client + encoding=_get_encoding(), + ) + if stream is True: + return CustomStreamWrapper( + completion_stream=response, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging, + ) - _response = openai_text_completions.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, # type: ignore[arg-type] - custom_llm_provider="text-completion-inception", - api_base=api_base, - acompletion=acompletion, - client=client, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - ) + return response - if ( - optional_params.get("stream", False) is False - and acompletion is False - and text_completion is False - ): - _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( - response_object=_response, model_response_object=model_response - ) - if optional_params.get("stream", False) or acompletion is True: - logging.post_call( - input=messages, - api_key=api_key, - original_response=_response, - additional_args={"headers": headers}, - ) - response = _response - elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): - # boto3 reads keys from .env - # sagemaker_chat: HF Messages API endpoints - # sagemaker_nova: Nova models on SageMaker (OpenAI-compatible) - model_response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) +def _complete_langgraph(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.langgraph.chat.transformation import LangGraphConfig - ## RESPONSE OBJECT - response = model_response - elif custom_llm_provider == "sagemaker": - # boto3 reads keys from .env - model_response = sagemaker_llm.completion( - model=model, - messages=messages, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - custom_prompt_dict=custom_prompt_dict, - hf_model_name=hf_model_name, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - acompletion=acompletion, - ) + ( + api_base, + api_key, + ) = LangGraphConfig()._get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) - ## RESPONSE OBJECT - response = model_response - elif custom_llm_provider == "bedrock": - # boto3 reads keys from .env - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict + headers = headers or litellm.headers - if "aws_bedrock_client" in optional_params: - verbose_logger.warning( - "'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication." - ) - # Extract credentials for legacy boto3 client and pass thru to httpx - aws_bedrock_client = optional_params.pop("aws_bedrock_client") - creds = aws_bedrock_client._get_credentials().get_frozen_credentials() - - if creds.access_key: - optional_params["aws_access_key_id"] = creds.access_key - if creds.secret_key: - optional_params["aws_secret_access_key"] = creds.secret_key - if creds.token: - optional_params["aws_session_token"] = creds.token - if ( - "aws_region_name" not in optional_params - or optional_params["aws_region_name"] is None - ): - optional_params["aws_region_name"] = ( - aws_bedrock_client.meta.region_name - ) + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) - bedrock_route = BedrockModelInfo.get_bedrock_route(model) - if bedrock_route == "claude_platform": - provider_config = ProviderConfigManager.get_provider_chat_config( - model=model, - provider=LlmProviders.BEDROCK, - ) - model = BedrockModelInfo.get_claude_platform_model(model) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - provider_config=provider_config, - ) - return response - elif bedrock_route == "converse": - model = model.replace("converse/", "") - response = bedrock_converse_chat_completion.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, # type: ignore - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - extra_headers=headers, # Use merged headers instead of original extra_headers - timeout=timeout, - acompletion=acompletion, - client=client, - api_base=api_base, - api_key=api_key, - ) - elif bedrock_route == "converse_like": - model = model.replace("converse_like/", "") - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - else: - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - custom_llm_provider="bedrock", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) - elif custom_llm_provider == "watsonx": - response = watsonx_chat_completion.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - print_verbose=print_verbose, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, - client=client, # pass AsyncOpenAI, OpenAI client - encoding=_get_encoding(), - custom_llm_provider="watsonx", - ) - elif custom_llm_provider == "watsonx_text": - api_key = ( - api_key - or optional_params.pop("apikey", None) - or get_secret_str("WATSONX_APIKEY") - or get_secret_str("WATSONX_API_KEY") - or get_secret_str("WX_API_KEY") - ) + return response - api_base = ( - api_base - or optional_params.pop( - "url", - optional_params.pop( - "api_base", optional_params.pop("base_url", None) - ), - ) - or get_secret_str("WATSONX_API_BASE") - or get_secret_str("WATSONX_URL") - or get_secret_str("WX_URL") - or get_secret_str("WML_URL") - ) - wx_credentials = optional_params.pop( - "wx_credentials", - optional_params.pop( - "watsonx_credentials", None - ), # follow {provider}_credentials, same as vertex ai - ) +def _complete_langflow(ctx: _CompletionDispatchContext): + acompletion = ctx.acompletion + api_base = ctx.api_base + api_key = ctx.api_key + client = ctx.client + custom_llm_provider = ctx.custom_llm_provider + headers = ctx.headers + litellm_params = ctx.litellm_params + logging = ctx.logging + messages = ctx.messages + model = ctx.model + model_response = ctx.model_response + optional_params = ctx.optional_params + shared_session = ctx.shared_session + stream = ctx.stream + timeout = ctx.timeout + + from litellm.llms.langflow.chat.transformation import LangFlowConfig + + ( + api_base, + api_key, + ) = LangFlowConfig()._get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + + return response + + +@client +def completion( # type: ignore + model: str, + # Optional OpenAI params: see https://platform.openai.com/docs/api-reference/chat/create + messages: List = [], + timeout: Optional[Union[float, str, httpx.Timeout]] = None, + temperature: Optional[float] = None, + top_p: Optional[float] = None, + n: Optional[int] = None, + stream: Optional[bool] = None, + stream_options: Optional[dict] = None, + stop=None, + max_completion_tokens: Optional[int] = None, + max_tokens: Optional[int] = None, + modalities: Optional[List[ChatCompletionModality]] = None, + prediction: Optional[ChatCompletionPredictionContentParam] = None, + audio: Optional[ChatCompletionAudioParam] = None, + presence_penalty: Optional[float] = None, + frequency_penalty: Optional[float] = None, + logit_bias: Optional[dict] = None, + user: Optional[str] = None, + # openai v1.0+ new params + reasoning_effort: Optional[ + Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] + ] = None, + verbosity: Optional[Literal["low", "medium", "high"]] = None, + response_format: Optional[Union[dict, Type[BaseModel]]] = None, + seed: Optional[int] = None, + tools: Optional[List] = None, + tool_choice: Optional[Union[str, dict]] = None, + logprobs: Optional[bool] = None, + top_logprobs: Optional[int] = None, + parallel_tool_calls: Optional[bool] = None, + web_search_options: Optional[OpenAIWebSearchOptions] = None, + include_server_side_tool_invocations: Optional[bool] = None, + deployment_id=None, + extra_headers: Optional[dict] = None, + safety_identifier: Optional[str] = None, + service_tier: Optional[str] = None, + # soon to be deprecated params by OpenAI + functions: Optional[List] = None, + function_call: Optional[str] = None, + # set api_base, api_version, api_key + base_url: Optional[str] = None, + api_version: Optional[str] = None, + api_key: Optional[str] = None, + model_list: Optional[list] = None, # pass in a list of api_base,keys, etc. + # Optional liteLLM function params + thinking: Optional[AnthropicThinkingParam] = None, + # Session management + shared_session: Optional["ClientSession"] = None, + # Per-request JSON schema validation (overrides litellm.enable_json_schema_validation) + enable_json_schema_validation: Optional[bool] = None, + **kwargs, +) -> Union[ModelResponse, CustomStreamWrapper]: + """ + Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly) + Parameters: + model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/ + messages (List): A list of message objects representing the conversation context (default is an empty list). - token: Optional[str] = None - if wx_credentials is not None: - api_base = wx_credentials.get("url", api_base) - api_key = wx_credentials.get( - "apikey", wx_credentials.get("api_key", api_key) - ) - token = wx_credentials.get( - "token", - wx_credentials.get( - "watsonx_token", None - ), # follow format of {provider}_token, same as azure - e.g. 'azure_ad_token=..' - ) + OPTIONAL PARAMS + functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list). + function_call (str, optional): The name of the function to call within the conversation (default is an empty string). + temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0). + top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0). + n (int, optional): The number of completions to generate (default is 1). + stream (bool, optional): If True, return a streaming response (default is False). + stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true. + stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens. + max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity). + max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens. + modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request.. You can use `["text", "audio"]` + prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content. + audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"] + presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far. + frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far. + logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion. + user (str, optional): A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse. + logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message + top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used. + metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc. + api_base (str, optional): Base URL for the API (default is None). + api_version (str, optional): API version (default is None). + api_key (str, optional): API key (default is None). + model_list (list, optional): List of api base, version, keys + extra_headers (dict, optional): Additional headers to include in the request. - if token is not None: - optional_params["token"] = token + LITELLM Specific Params + mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None). + custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock" + max_retries (int, optional): The number of retries to attempt (default is 0). + Returns: + ModelResponse: A response object containing the generated completion and associated metadata. - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="watsonx_text", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) - elif custom_llm_provider == "vllm": - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - model_response = vllm_handler.completion( - model=model, - messages=messages, - custom_prompt_dict=custom_prompt_dict, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - ) + Note: + - This function is used to perform completions() using the specified language model. + - It supports various optional parameters for customizing the completion behavior. + - If 'mock_response' is provided, a mock completion response is returned for testing or debugging. + """ + ### VALIDATE Request ### + if model is None: + raise ValueError("model param not passed in.") + # validate messages + messages = validate_and_fix_openai_messages(messages=messages) + tools = validate_and_fix_openai_tools(tools=tools) + # validate tool_choice + tool_choice = validate_chat_completion_tool_choice(tool_choice=tool_choice) + # validate optional params + stop = validate_openai_optional_params(stop=stop) + # normalize camelCase thinking keys (e.g. budgetTokens -> budget_tokens) + thinking = validate_and_fix_thinking_param(thinking=thinking) - if ( - "stream" in optional_params and optional_params["stream"] is True - ): ## [BETA] - # don't try to access stream object, - response = CustomStreamWrapper( - model_response, - model, - custom_llm_provider="vllm", - logging_obj=logging, - ) - return response + ######### unpacking kwargs ##################### + args = locals() - ## RESPONSE OBJECT - response = model_response - elif custom_llm_provider == "ollama": - api_base = ( - litellm.api_base - or api_base - or get_secret("OLLAMA_API_BASE") - or "http://localhost:11434" - ) - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" + skip_mcp_handler = kwargs.pop("_skip_mcp_handler", False) + if not skip_mcp_handler and tools: + from litellm.responses.mcp.chat_completions_handler import acompletion_with_mcp + from litellm.responses.mcp.litellm_proxy_mcp_handler import ( + LiteLLM_Proxy_MCP_Handler, + ) + from litellm.types.llms.openai import ToolParam - response = base_llm_http_handler.completion( + # Check if MCP tools are present (following responses pattern) + # Cast tools to Optional[Iterable[ToolParam]] for type checking + tools_for_mcp = cast(Optional[Iterable[ToolParam]], tools) + if LiteLLM_Proxy_MCP_Handler._should_use_litellm_mcp_gateway( + tools=tools_for_mcp + ): + # Return coroutine - acompletion will await it + # completion() can return a coroutine when MCP tools are present, which acompletion() awaits + return acompletion_with_mcp( # type: ignore[return-value] model=model, - stream=stream, messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama", + functions=functions, + function_call=function_call, timeout=timeout, - headers=headers, - encoding=_get_encoding(), + temperature=temperature, + top_p=top_p, + n=n, + stream=stream, + stream_options=stream_options, + stop=stop, + max_tokens=max_tokens, + max_completion_tokens=max_completion_tokens, + modalities=modalities, + prediction=prediction, + audio=audio, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + logit_bias=logit_bias, + user=user, + response_format=response_format, + seed=seed, + tools=tools, + tool_choice=tool_choice, + parallel_tool_calls=parallel_tool_calls, + logprobs=logprobs, + top_logprobs=top_logprobs, + deployment_id=deployment_id, + reasoning_effort=reasoning_effort, + verbosity=verbosity, + safety_identifier=safety_identifier, + service_tier=service_tier, + base_url=base_url, + api_version=api_version, api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, + model_list=model_list, + extra_headers=extra_headers, + thinking=thinking, + web_search_options=web_search_options, + shared_session=shared_session, + enable_json_schema_validation=enable_json_schema_validation, + **kwargs, ) + api_base = kwargs.get("api_base", None) + mock_response: Optional[MOCK_RESPONSE_TYPE] = kwargs.get("mock_response", None) + mock_tool_calls = kwargs.get("mock_tool_calls", None) + mock_timeout = cast(Optional[bool], kwargs.get("mock_timeout", None)) + force_timeout = kwargs.get("force_timeout", 600) ## deprecated + logger_fn = kwargs.get("logger_fn", None) + verbose = kwargs.get("verbose", False) + custom_llm_provider = kwargs.get("custom_llm_provider", None) + litellm_logging_obj = kwargs.get("litellm_logging_obj", None) + id = kwargs.get("id", None) + metadata = kwargs.get("metadata", None) + model_info = kwargs.get("model_info", None) + proxy_server_request = kwargs.get("proxy_server_request", None) + fallbacks = kwargs.get("fallbacks", None) + provider_specific_header = cast( + Optional[ProviderSpecificHeader], kwargs.get("provider_specific_header", None) + ) + headers = kwargs.get("headers", None) or extra_headers - elif custom_llm_provider == "ollama_chat": - api_base = ( - litellm.api_base - or api_base - or get_secret("OLLAMA_API_BASE") - or "http://localhost:11434" - ) + ensure_alternating_roles: Optional[bool] = kwargs.get( + "ensure_alternating_roles", None + ) + user_continue_message: Optional[ChatCompletionUserMessage] = kwargs.get( + "user_continue_message", None + ) + assistant_continue_message: Optional[ChatCompletionAssistantMessage] = kwargs.get( + "assistant_continue_message", None + ) + if headers is None: + headers = {} + if extra_headers is not None: + headers.update(extra_headers) + # Inject proxy auth headers if configured + if litellm.proxy_auth is not None: + try: + proxy_headers = litellm.proxy_auth.get_auth_headers() + headers.update(proxy_headers) + except Exception as e: + verbose_logger.warning(f"Failed to get proxy auth headers: {e}") + num_retries = kwargs.get( + "num_retries", None + ) ## alt. param for 'max_retries'. Use this to pass retries w/ instructor. + max_retries = kwargs.get("max_retries", None) + cooldown_time = kwargs.get("cooldown_time", None) + context_window_fallback_dict = kwargs.get("context_window_fallback_dict", None) + organization = kwargs.get("organization", None) + ### VERIFY SSL ### + ssl_verify = kwargs.get("ssl_verify", None) + ### CUSTOM MODEL COST ### + input_cost_per_token = kwargs.get("input_cost_per_token", None) + output_cost_per_token = kwargs.get("output_cost_per_token", None) + input_cost_per_second = kwargs.get("input_cost_per_second", None) + output_cost_per_second = kwargs.get("output_cost_per_second", None) + ### CUSTOM PROMPT TEMPLATE ### + initial_prompt_value = kwargs.get("initial_prompt_value", None) + roles = kwargs.get("roles", None) + final_prompt_value = kwargs.get("final_prompt_value", None) + bos_token = kwargs.get("bos_token", None) + eos_token = kwargs.get("eos_token", None) + preset_cache_key = kwargs.get("preset_cache_key", None) + hf_model_name = kwargs.get("hf_model_name", None) + supports_system_message = kwargs.get("supports_system_message", None) + base_model = kwargs.get("base_model", None) or ( + model_info.get("base_model") if isinstance(model_info, dict) else None + ) + ### DISABLE FLAGS ### + disable_add_transform_inline_image_block = kwargs.get( + "disable_add_transform_inline_image_block", None + ) + ### TEXT COMPLETION CALLS ### + text_completion = kwargs.get("text_completion", False) + atext_completion = kwargs.get("atext_completion", False) + ### ASYNC CALLS ### + acompletion = kwargs.get("acompletion", False) + client = kwargs.get("client", None) + ### Admin Controls ### + no_log = kwargs.get("no-log", False) + ### PROMPT MANAGEMENT ### + prompt_id = cast(Optional[str], kwargs.get("prompt_id", None)) + prompt_variables = cast(Optional[dict], kwargs.get("prompt_variables", None)) + litellm_system_prompt = kwargs.get("litellm_system_prompt", None) + ### COPY MESSAGES ### - related issue https://github.com/BerriAI/litellm/discussions/4489 + messages = get_completion_messages( + messages=messages, + ensure_alternating_roles=ensure_alternating_roles or False, + user_continue_message=user_continue_message, + assistant_continue_message=assistant_continue_message, + ) + ######## end of unpacking kwargs ########### + non_default_params = get_non_default_completion_params(kwargs=kwargs) + litellm_params = {} # used to prevent unbound var errors + ## PROMPT MANAGEMENT HOOKS ## - api_key = ( - api_key - or litellm.ollama_key - or os.environ.get("OLLAMA_API_KEY") - or litellm.api_key - ) - if api_key is not None and "Authorization" not in headers: - headers["Authorization"] = f"Bearer {api_key}" + if isinstance(litellm_logging_obj, LiteLLMLoggingObj) and ( + litellm_logging_obj.should_run_prompt_management_hooks( + prompt_id=prompt_id, non_default_params=non_default_params + ) + ): + ( + model, + messages, + optional_params, + ) = litellm_logging_obj.get_chat_completion_prompt( + model=model, + messages=messages, + non_default_params=non_default_params, + prompt_id=prompt_id, + prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), + prompt_version=kwargs.get("prompt_version", None), + ) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="ollama_chat", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - client=client, - ) + ### LITELLM SYSTEM PROMPT ### + if litellm_system_prompt: + messages = add_system_prompt_to_messages( + messages=messages, + system_prompt=litellm_system_prompt, + merge_with_first_system=True, + ) - elif custom_llm_provider == "triton": - api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - ) - elif custom_llm_provider == "cloudflare": - api_key = ( - api_key - or litellm.cloudflare_api_key - or litellm.api_key - or get_secret("CLOUDFLARE_API_KEY") - ) - account_id = get_secret("CLOUDFLARE_ACCOUNT_ID") - api_base = ( - api_base - or litellm.api_base - or get_secret("CLOUDFLARE_API_BASE") - or f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/" - ) + try: + if base_url is not None: + api_base = base_url + if num_retries is not None: + max_retries = num_retries + logging: LiteLLMLoggingObj = cast(LiteLLMLoggingObj, litellm_logging_obj) + fallbacks = fallbacks or litellm.model_fallbacks + if fallbacks is not None: + return completion_with_fallbacks(**args) + if model_list is not None: + deployments = [ + m["litellm_params"] for m in model_list if m["model_name"] == model + ] + return litellm.batch_completion_models(deployments=deployments, **args) + if litellm.model_alias_map and model in litellm.model_alias_map: + model = litellm.model_alias_map[ + model + ] # update the model to the actual value if an alias has been passed in + model_response = ModelResponse() + setattr(model_response, "usage", litellm.Usage()) + if ( + kwargs.get("azure", False) is True + ): # don't remove flag check, to remain backwards compatible for repos like Codium + custom_llm_provider = "azure" + if deployment_id is not None: # azure llms + model = deployment_id + custom_llm_provider = "azure" + _supplemental_provider_params = { + k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs + } + model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + litellm_params=( + GenericLiteLLMParams(**_supplemental_provider_params) + if _supplemental_provider_params + else None + ), + ) - custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="cloudflare", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements - ) + ## RESPONSES API BRIDGE LOGIC ## - check early and normalize model name + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + ) - elif custom_llm_provider == "petals" or model in litellm.petals_models: - api_base = api_base or litellm.api_base + if not _should_allow_input_examples( + custom_llm_provider=custom_llm_provider, model=model + ): + tools = _drop_input_examples_from_tools(tools=tools) - custom_llm_provider = "petals" - stream = optional_params.pop("stream", False) - model_response = petals_handler.completion( - model=model, - messages=messages, - api_base=api_base, - model_response=model_response, - print_verbose=print_verbose, - optional_params=optional_params, - litellm_params=litellm_params, - logger_fn=logger_fn, - encoding=_get_encoding(), - logging_obj=logging, - client=client, - ) - if stream is True: ## [BETA] - # Fake streaming for petals - resp_string = model_response["choices"][0]["message"]["content"] - response = CustomStreamWrapper( - resp_string, - model, - custom_llm_provider="petals", - logging_obj=logging, - ) - return response - response = model_response - elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: - try: - client = ( - HTTPHandler(timeout=timeout) if stream is False else None - ) # Keep this here, otherwise, the httpx.client closes and streaming is impossible - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - timeout=timeout, # type: ignore - client=client, + if provider_specific_header is not None: + headers.update( + ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=provider_specific_header, custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, ) - - except Exception as e: - ## LOGGING - log the original exception returned - logging.post_call( - input=messages, - api_key=api_key, - original_response=str(e), - additional_args={"headers": headers}, - ) - raise e - elif custom_llm_provider == "gradient_ai": - api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider="gradient_ai", - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, ) - elif custom_llm_provider == "bytez": - api_key = ( - api_key - or litellm.bytez_key - or get_secret_str("BYTEZ_API_KEY") - or litellm.api_key + if model_response is not None and hasattr(model_response, "_hidden_params"): + model_response._hidden_params["custom_llm_provider"] = custom_llm_provider + model_response._hidden_params["region_name"] = kwargs.get( + "aws_region_name", None + ) # support region-based pricing for bedrock + + ### TIMEOUT LOGIC ### + timeout = CompletionTimeout.resolve( + timeout, + kwargs, + custom_llm_provider, + global_timeout=getattr(litellm, "request_timeout", None), + supports_httpx_timeout=supports_httpx_timeout, + ) + + ### REGISTER CUSTOM MODEL PRICING -- IF GIVEN ### + if ( + input_cost_per_token is not None and output_cost_per_token is not None + ) or input_cost_per_second is not None: + litellm.register_model( + { + f"{custom_llm_provider}/{model}": _build_custom_pricing_entry( + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + model_info=model_info, + ) + } ) + ### BUILD CUSTOM PROMPT TEMPLATE -- IF GIVEN ### + custom_prompt_dict = {} # type: ignore + if ( + initial_prompt_value + or roles + or final_prompt_value + or bos_token + or eos_token + ): + custom_prompt_dict = {model: {}} + if initial_prompt_value: + custom_prompt_dict[model]["initial_prompt_value"] = initial_prompt_value + if roles: + custom_prompt_dict[model]["roles"] = roles + if final_prompt_value: + custom_prompt_dict[model]["final_prompt_value"] = final_prompt_value + if bos_token: + custom_prompt_dict[model]["bos_token"] = bos_token + if eos_token: + custom_prompt_dict[model]["eos_token"] = eos_token - response = base_llm_http_handler.completion( + messages = update_messages_with_model_file_ids( + messages=messages, + model_id=kwargs.get("model_info", {}).get("id", None), + model_file_id_mapping=cast( + Dict[str, Dict[str, str]], + kwargs.get("model_file_id_mapping") or {}, + ), + ) + + provider_config: Optional[BaseConfig] = None + if custom_llm_provider is not None and custom_llm_provider in [ + provider.value for provider in LlmProviders + ]: + provider_config = ProviderConfigManager.get_provider_chat_config( model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=bytez_transformation, + provider=LlmProviders(custom_llm_provider), + base_model=base_model, ) - pass - elif custom_llm_provider == "lemonade": - api_key = ( - api_key - or litellm.lemonade_key - or get_secret_str("LEMONADE_API_KEY") - or litellm.api_key + if provider_config is not None: + messages = provider_config.translate_developer_role_to_system_role( + messages=messages ) - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, - acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, - custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=lemonade_transformation, - ) + if ( + supports_system_message is not None + and isinstance(supports_system_message, bool) + and supports_system_message is False + ): + messages = map_system_message_pt(messages=messages) - pass + if dynamic_api_key is not None: + api_key = dynamic_api_key + # check if user passed in any of the OpenAI optional params + optional_param_args = { + "functions": functions, + "function_call": function_call, + "temperature": temperature, + "top_p": top_p, + "n": n, + "stream": stream, + "stream_options": stream_options, + "stop": stop, + "max_tokens": max_tokens, + "max_completion_tokens": max_completion_tokens, + "modalities": modalities, + "prediction": prediction, + "audio": audio, + "presence_penalty": presence_penalty, + "frequency_penalty": frequency_penalty, + "logit_bias": logit_bias, + "user": user, + # params to identify the model + "model": model, + "custom_llm_provider": custom_llm_provider, + "response_format": response_format, + "seed": seed, + "tools": tools, + "tool_choice": tool_choice, + "max_retries": max_retries, + "logprobs": logprobs, + "top_logprobs": top_logprobs, + "api_version": api_version, + "parallel_tool_calls": parallel_tool_calls, + "messages": messages, + "reasoning_effort": reasoning_effort, + "thinking": thinking, + "web_search_options": web_search_options, + "include_server_side_tool_invocations": ( + include_server_side_tool_invocations + if include_server_side_tool_invocations is not None + else kwargs.get("include_server_side_tool_invocations") + ), + "safety_identifier": safety_identifier, + "service_tier": service_tier, + "allowed_openai_params": kwargs.get("allowed_openai_params"), + "base_model": base_model, + } + optional_params = get_optional_params( + **optional_param_args, **non_default_params + ) + processed_non_default_params = pre_process_non_default_params( + model=model, + passed_params=optional_param_args, + special_params=non_default_params, + custom_llm_provider=custom_llm_provider, + additional_drop_params=kwargs.get("additional_drop_params"), + remove_sensitive_keys=True, + add_provider_specific_params=True, + provider_config=provider_config, + ) - elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: - api_key = ( - api_key - or litellm.ovhcloud_key - or get_secret_str("OVHCLOUD_API_KEY") - or litellm.api_key + if litellm.add_function_to_prompt and optional_params.get( + "functions_unsupported_model", None + ): # if user opts to add it to prompt, when API doesn't support function calling + functions_unsupported_model = optional_params.pop( + "functions_unsupported_model" ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("OVHCLOUD_API_BASE") - or "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" + messages = function_call_prompt( + messages=messages, functions=functions_unsupported_model ) - response = base_llm_http_handler.completion( - model=model, - messages=messages, - headers=headers, - model_response=model_response, - api_key=api_key, - api_base=api_base, + # For logging - save the values of the litellm-specific params passed in + litellm_params = get_litellm_params( + acompletion=acompletion, + api_key=api_key, + force_timeout=force_timeout, + logger_fn=logger_fn, + verbose=verbose, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + litellm_call_id=kwargs.get("litellm_call_id", None), + model_alias_map=litellm.model_alias_map, + completion_call_id=id, + metadata=metadata, + model_info=model_info, + proxy_server_request=proxy_server_request, + preset_cache_key=preset_cache_key, + no_log=no_log, + input_cost_per_second=input_cost_per_second, + input_cost_per_token=input_cost_per_token, + output_cost_per_second=output_cost_per_second, + output_cost_per_token=output_cost_per_token, + cooldown_time=cooldown_time, + text_completion=kwargs.get("text_completion"), + azure_ad_token_provider=kwargs.get("azure_ad_token_provider"), + user_continue_message=kwargs.get("user_continue_message"), + base_model=base_model, + litellm_trace_id=kwargs.get("litellm_trace_id"), + litellm_session_id=kwargs.get("litellm_session_id"), + hf_model_name=hf_model_name, + custom_prompt_dict=custom_prompt_dict, + litellm_metadata=kwargs.get("litellm_metadata"), + disable_add_transform_inline_image_block=disable_add_transform_inline_image_block, + drop_params=kwargs.get("drop_params"), + prompt_id=prompt_id, + prompt_variables=prompt_variables, + ssl_verify=ssl_verify, + merge_reasoning_content_in_choices=kwargs.get( + "merge_reasoning_content_in_choices", None + ), + use_litellm_proxy=kwargs.get("use_litellm_proxy", False), + api_version=api_version, + azure_ad_token=kwargs.get("azure_ad_token"), + tenant_id=kwargs.get("tenant_id"), + client_id=kwargs.get("client_id"), + client_secret=kwargs.get("client_secret"), + azure_username=kwargs.get("azure_username"), + azure_password=kwargs.get("azure_password"), + azure_scope=kwargs.get("azure_scope"), + max_retries=max_retries, + timeout=timeout, + litellm_request_debug=kwargs.get("litellm_request_debug", False), + tpm=kwargs.get("tpm"), + rpm=kwargs.get("rpm"), + use_xai_oauth=kwargs.get("use_xai_oauth", False), + aws_bedrock_project_id=kwargs.get("aws_bedrock_project_id"), + ) + cast(LiteLLMLoggingObj, logging).update_environment_variables( + model=model, + user=user, + optional_params=processed_non_default_params, # [IMPORTANT] - using processed_non_default_params ensures consistent params logged to langfuse for finetuning / eval datasets. + litellm_params=litellm_params, + custom_llm_provider=custom_llm_provider, + ) + if mock_response or mock_tool_calls or mock_timeout: + kwargs.pop("mock_timeout", None) # remove for any fallbacks triggered + return mock_completion( + model, + messages, + stream=stream, + n=n, + mock_response=mock_response, + mock_tool_calls=mock_tool_calls, + logging=logging, acompletion=acompletion, - logging_obj=logging, - optional_params=optional_params, - litellm_params=litellm_params, - timeout=timeout, # type: ignore - client=client, + mock_delay=kwargs.get("mock_delay", None), custom_llm_provider=custom_llm_provider, - encoding=_get_encoding(), - stream=stream, - provider_config=ovhcloud_transformation, + mock_timeout=mock_timeout, + timeout=timeout, ) - pass - - elif custom_llm_provider == "custom": - url = litellm.api_base or api_base or "" - if url is None or url == "": - raise ValueError( - "api_base not set. Set api_base or litellm.api_base for custom endpoints" - ) - - """ - assume input to custom LLM api bases follow this format: - resp = litellm.module_level_client.post( - api_base, - json={ - 'model': 'meta-llama/Llama-2-13b-hf', # model name - 'params': { - 'prompt': ["The capital of France is P"], - 'max_tokens': 32, - 'temperature': 0.7, - 'top_p': 1.0, - 'top_k': 40, - } - } + ## RESPONSES API BRIDGE LOGIC ## - check if model has 'mode: responses' in litellm.model_cost map + # Only run the second bridge check if the first one didn't already + # detect responses mode (e.g. via the "responses/" prefix). The second + # check handles cases like gpt-5.4+ with tools+reasoning_effort or + # reasoningSummary/reasoning_summary without tools (AI SDK) that the first + # (early) check doesn't cover. + _reasoning_summary_for_bridge = peek_reasoning_summary_aliases(optional_params) + if responses_api_model_info.get("mode") != "responses": + responses_api_model_info, model = responses_api_bridge_check( + model=model, + custom_llm_provider=custom_llm_provider, + web_search_options=web_search_options, + tools=tools, + reasoning_effort=reasoning_effort, + reasoning_summary=_reasoning_summary_for_bridge, ) - """ - prompt = " ".join([message["content"] for message in messages]) # type: ignore - resp = litellm.module_level_client.post( - url, - headers=headers, - json={ - "model": model, - "params": { - "prompt": [prompt], - "max_tokens": max_tokens, - "temperature": temperature, - "top_p": top_p, - "top_k": kwargs.get("top_k"), - }, - **kwargs.get("extra_body", {}), - }, - ) - response_json = resp.json() - """ - assume all responses from custom api_bases of this format: - { - 'data': [ - { - 'prompt': 'The capital of France is P', - 'output': ['The capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France is PARIS.\nThe capital of France'], - 'params': {'temperature': 0.7, 'top_k': 40, 'top_p': 1}}], - 'message': 'ok' - } - ] - } - """ - string_response = response_json["data"][0]["output"][0] - ## RESPONSE OBJECT - model_response.choices[0].message.content = string_response # type: ignore - model_response.created = int(time.time()) - model_response.model = model - response = model_response - - elif ( - custom_llm_provider in litellm._custom_providers - ): # Assume custom LLM provider - # Get the Custom Handler - custom_handler: Optional[CustomLLM] = None - for item in litellm.custom_provider_map: - if item["provider"] == custom_llm_provider: - custom_handler = item["custom_handler"] + # Use base_model (the true underlying model) for Azure model-type + # detection when the deployment name differs from the model name. + _azure_detection_model = base_model or model - if custom_handler is None: - raise LiteLLMUnknownProvider( - model=model, custom_llm_provider=custom_llm_provider - ) + if responses_api_model_info.get("mode") == "responses": + from litellm.completion_extras import responses_api_bridge - ## ROUTE LLM CALL ## - handler_fn = custom_chat_llm_router( - async_fn=acompletion, stream=stream, custom_llm=custom_handler + optional_params, rs_val = ( + strip_reasoning_summary_aliases_from_optional_params(optional_params) ) - headers = headers or litellm.headers or {} + if isinstance(reasoning_effort, dict) and "summary" in reasoning_effort: + optional_params["reasoning_effort"] = reasoning_effort + elif rs_val is not None: + eff = optional_params.get("reasoning_effort", reasoning_effort) + if isinstance(eff, dict): + optional_params["reasoning_effort"] = {**eff, "summary": rs_val} + elif eff is not None: + optional_params["reasoning_effort"] = { + "effort": eff, + "summary": rs_val, + } + else: + optional_params["reasoning_effort"] = {"summary": rs_val} - ## CALL FUNCTION - response = handler_fn( + return responses_api_bridge.completion( model=model, messages=messages, headers=headers, model_response=model_response, - print_verbose=print_verbose, api_key=api_key, api_base=api_base, acompletion=acompletion, logging_obj=logging, optional_params=optional_params, litellm_params=litellm_params, - logger_fn=logger_fn, timeout=timeout, # type: ignore - custom_prompt_dict=custom_prompt_dict, client=client, # pass AsyncOpenAI, OpenAI client + custom_llm_provider=custom_llm_provider, encoding=_get_encoding(), + stream=stream, + ) + elif ( + custom_llm_provider == "openai" + and OpenAIGPT5Config.is_model_gpt_5_model(model) + ) or ( + custom_llm_provider == "azure" + and litellm.AzureOpenAIGPT5Config.is_model_gpt_5_model( + _azure_detection_model + ) + ): + optional_params, _ = strip_reasoning_summary_aliases_from_optional_params( + optional_params ) - if stream is True: - return CustomStreamWrapper( - completion_stream=response, - model=model, - custom_llm_provider=custom_llm_provider, - logging_obj=logging, - ) - elif custom_llm_provider == "langgraph": - # LangGraph - Agent Runtime Provider - from litellm.llms.langgraph.chat.transformation import LangGraphConfig + _dispatch_ctx = _CompletionDispatchContext( + _azure_detection_model=_azure_detection_model, + acompletion=acompletion, + api_base=api_base, + api_key=api_key, + api_version=api_version, + client=client, + custom_llm_provider=custom_llm_provider, + custom_prompt_dict=custom_prompt_dict, + extra_headers=extra_headers, + headers=headers, + hf_model_name=hf_model_name, + kwargs=kwargs, + litellm_params=litellm_params, + logger_fn=logger_fn, + logging=logging, + max_retries=max_retries, + max_tokens=max_tokens, + messages=messages, + metadata=metadata, + model=model, + model_response=model_response, + optional_params=optional_params, + organization=organization, + provider_config=provider_config, + shared_session=shared_session, + stream=stream, + temperature=temperature, + text_completion=text_completion, + timeout=timeout, + top_p=top_p, + ) + if custom_llm_provider == "azure": + # azure configs + ## check dynamic params ## + response = _complete_azure(_dispatch_ctx) + elif custom_llm_provider == "azure_text": + # azure configs + response = _complete_azure_text(_dispatch_ctx) + elif custom_llm_provider == "deepseek": + ## COMPLETION CALL - ( - api_base, - api_key, - ) = LangGraphConfig()._get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, - ) + response = _complete_deepseek(_dispatch_ctx) - headers = headers or litellm.headers + elif custom_llm_provider == "azure_ai": + response = _complete_azure_ai(_dispatch_ctx) + elif ( + custom_llm_provider == "text-completion-openai" + or "ft:babbage-002" in model + or "ft:davinci-002" in model # support for finetuned completion models + or custom_llm_provider + in litellm.openai_text_completion_compatible_providers + and kwargs.get("text_completion") is True + ): + response = _complete_text_completion_openai(_dispatch_ctx) + elif custom_llm_provider == "fireworks_ai": + ## COMPLETION CALL + response = _complete_fireworks_ai(_dispatch_ctx) + elif custom_llm_provider == "heroku": + response = _complete_heroku(_dispatch_ctx) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + elif custom_llm_provider == "ragflow": + ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths + response = _complete_ragflow(_dispatch_ctx) + elif custom_llm_provider == "xai": + ## COMPLETION CALL + response = _complete_xai(_dispatch_ctx) + elif custom_llm_provider == "groq": + response = _complete_groq(_dispatch_ctx) + elif custom_llm_provider == "bedrock_mantle": + response = _complete_bedrock_mantle(_dispatch_ctx) + elif custom_llm_provider == "a2a": + # A2A (Agent-to-Agent) Protocol + # Resolve agent configuration from registry if model format is "a2a/" + response = _complete_a2a(_dispatch_ctx) + elif custom_llm_provider == "gigachat": + # GigaChat - Sber AI's LLM (Russia) + response = _complete_gigachat(_dispatch_ctx) - elif custom_llm_provider == "langflow": - # LangFlow - Visual AI Agent Platform - from litellm.llms.langflow.chat.transformation import LangFlowConfig + elif custom_llm_provider == "sap": + response = _complete_sap(_dispatch_ctx) + elif custom_llm_provider == "aiohttp_openai": + # NEW aiohttp provider for 10-100x higher RPS + response = _complete_aiohttp_openai(_dispatch_ctx) + elif custom_llm_provider == "cometapi": + response = _complete_cometapi(_dispatch_ctx) + elif custom_llm_provider == "minimax": + response = _complete_minimax(_dispatch_ctx) + elif custom_llm_provider == "hosted_vllm": + response = _complete_hosted_vllm(_dispatch_ctx) + elif ( + model in litellm.open_ai_chat_completion_models + or custom_llm_provider == "custom_openai" + or custom_llm_provider == "deepinfra" + or custom_llm_provider == "perplexity" + or custom_llm_provider == "nvidia_nim" + or custom_llm_provider == "cerebras" + or custom_llm_provider == "baseten" + or custom_llm_provider == "sambanova" + or custom_llm_provider == "volcengine" + or custom_llm_provider == "anyscale" + or custom_llm_provider == "openai" + or custom_llm_provider == "together_ai" + or custom_llm_provider == "nebius" + or custom_llm_provider == "wandb" + or custom_llm_provider == "clarifai" + or custom_llm_provider in litellm.openai_compatible_providers + or JSONProviderRegistry.exists( + custom_llm_provider + ) # JSON-configured providers + or "ft:gpt-3.5-turbo" in model # finetune gpt-3.5-turbo + ): # allow user to make an openai call with a custom base + # note: if a user sets a custom base - we should ensure this works + # allow for the setting of dynamic and stateful api-bases + response = _complete_custom_openai(_dispatch_ctx) + + elif custom_llm_provider == "mistral": + response = _complete_mistral(_dispatch_ctx) + elif ( + "replicate" in model + or custom_llm_provider == "replicate" + or model in litellm.replicate_models + ): + # Setting the relevant API KEY for replicate, replicate defaults to using os.environ.get("REPLICATE_API_TOKEN") + response = _complete_replicate(_dispatch_ctx) + elif ( + "clarifai" in model + or custom_llm_provider == "clarifai" + or model in litellm.clarifai_models + ): + pass # Deprecated - handled in the openai compatible provider section above + elif custom_llm_provider == "anthropic_text": + response = _complete_anthropic_text(_dispatch_ctx) + elif custom_llm_provider == "anthropic": + response = _complete_anthropic(_dispatch_ctx) + elif custom_llm_provider == "nlp_cloud": + response = _complete_nlp_cloud(_dispatch_ctx) + elif custom_llm_provider == "aleph_alpha": + response = _complete_aleph_alpha(_dispatch_ctx) + elif custom_llm_provider == "cohere_chat" or custom_llm_provider == "cohere": + response = _complete_cohere_chat(_dispatch_ctx) + elif custom_llm_provider == "maritalk": + response = _complete_maritalk(_dispatch_ctx) + elif custom_llm_provider == "amazon_nova": + response = _complete_amazon_nova(_dispatch_ctx) + elif custom_llm_provider == "huggingface": + response = _complete_huggingface(_dispatch_ctx) + elif custom_llm_provider == "oci": + response = _complete_oci(_dispatch_ctx) + elif custom_llm_provider == "compactifai": + response = _complete_compactifai(_dispatch_ctx) + elif custom_llm_provider == "oobabooga": + response = _complete_oobabooga(_dispatch_ctx) + elif custom_llm_provider == "databricks": + response = _complete_databricks(_dispatch_ctx) - ( - api_base, - api_key, - ) = LangFlowConfig()._get_openai_compatible_provider_info( - api_base=api_base or litellm.api_base, - api_key=api_key or litellm.api_key, + elif custom_llm_provider == "datarobot": + response = _complete_datarobot(_dispatch_ctx) + elif custom_llm_provider == "openrouter": + response = _complete_openrouter(_dispatch_ctx) + elif custom_llm_provider == "vercel_ai_gateway": + response = _complete_vercel_ai_gateway(_dispatch_ctx) + elif ( + custom_llm_provider == "together_ai" + or ("togethercomputer" in model) + or (model in litellm.together_ai_models) + ): + """ + Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility + """ + pass + elif custom_llm_provider == "palm": + raise ValueError( + "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en" ) + elif custom_llm_provider == "vertex_ai_beta" or custom_llm_provider == "gemini": + response = _complete_vertex_ai_beta(_dispatch_ctx) - headers = headers or litellm.headers + elif custom_llm_provider == "vertex_ai": + response = _complete_vertex_ai(_dispatch_ctx) + elif custom_llm_provider == "predibase": + response = _complete_predibase(_dispatch_ctx) + elif custom_llm_provider == "text-completion-codestral": + response = _complete_text_completion_codestral(_dispatch_ctx) + elif custom_llm_provider == "text-completion-inception": + response = _complete_text_completion_inception(_dispatch_ctx) + elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): + # boto3 reads keys from .env + # sagemaker_chat: HF Messages API endpoints + # sagemaker_nova: Nova models on SageMaker (OpenAI-compatible) + response = _complete_sagemaker_chat(_dispatch_ctx) + elif custom_llm_provider == "sagemaker": + # boto3 reads keys from .env + response = _complete_sagemaker(_dispatch_ctx) + elif custom_llm_provider == "bedrock": + # boto3 reads keys from .env + response = _complete_bedrock(_dispatch_ctx) + elif custom_llm_provider == "watsonx": + response = _complete_watsonx(_dispatch_ctx) + elif custom_llm_provider == "watsonx_text": + response = _complete_watsonx_text(_dispatch_ctx) + elif custom_llm_provider == "vllm": + response = _complete_vllm(_dispatch_ctx) + elif custom_llm_provider == "ollama": + response = _complete_ollama(_dispatch_ctx) - response = base_llm_http_handler.completion( - model=model, - stream=stream, - messages=messages, - acompletion=acompletion, - api_base=api_base, - model_response=model_response, - optional_params=optional_params, - litellm_params=litellm_params, - shared_session=shared_session, - custom_llm_provider=custom_llm_provider, - timeout=timeout, - headers=headers, - encoding=_get_encoding(), - api_key=api_key, - logging_obj=logging, - client=client, - ) + elif custom_llm_provider == "ollama_chat": + response = _complete_ollama_chat(_dispatch_ctx) + + elif custom_llm_provider == "triton": + response = _complete_triton(_dispatch_ctx) + elif custom_llm_provider == "cloudflare": + response = _complete_cloudflare(_dispatch_ctx) + + elif custom_llm_provider == "petals" or model in litellm.petals_models: + response = _complete_petals(_dispatch_ctx) + elif custom_llm_provider == "snowflake" or model in litellm.snowflake_models: + response = _complete_snowflake(_dispatch_ctx) + elif custom_llm_provider == "gradient_ai": + response = _complete_gradient_ai(_dispatch_ctx) + + elif custom_llm_provider == "bytez": + response = _complete_bytez(_dispatch_ctx) + elif custom_llm_provider == "lemonade": + response = _complete_lemonade(_dispatch_ctx) + + elif custom_llm_provider == "ovhcloud" or model in litellm.ovhcloud_models: + response = _complete_ovhcloud(_dispatch_ctx) + + elif custom_llm_provider == "custom": + response = _complete_custom(_dispatch_ctx) + + elif ( + custom_llm_provider in litellm._custom_providers + ): # Assume custom LLM provider + # Get the Custom Handler + response = _complete_custom_providers(_dispatch_ctx) + + elif custom_llm_provider == "langgraph": + # LangGraph - Agent Runtime Provider + response = _complete_langgraph(_dispatch_ctx) + + elif custom_llm_provider == "langflow": + # LangFlow - Visual AI Agent Platform + response = _complete_langflow(_dispatch_ctx) else: raise LiteLLMUnknownProvider( From 20bcd775c4ab2cd48751f54b9ee980a29caa9109 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 19 Jun 2026 06:00:05 +0000 Subject: [PATCH 2/5] fix(completion): keep @tracer.wrap() on completion, not the dispatch context The dispatch-extraction refactor inserted _CompletionDispatchContext and its helpers between `return entry` and `completion`, which left completion's @tracer.wrap() decorator stranded on the new frozen dataclass. That silently dropped the tracing span from completion, the library's hottest entrypoint, and instead had ddtrace wrap the dataclass constructor (replacing the class with a function wrapper at runtime). Restore the decorator above @client on completion, matching litellm_internal_staging, and remove it from the dataclass. --- litellm/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/main.py b/litellm/main.py index b41a222bfe46..9f69d9c9875c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1085,7 +1085,6 @@ def _build_custom_pricing_entry( return entry -@tracer.wrap() @dataclass(frozen=True, slots=True) class _CompletionDispatchContext: _azure_detection_model: str @@ -5011,6 +5010,7 @@ def _complete_langflow(ctx: _CompletionDispatchContext): return response +@tracer.wrap() @client def completion( # type: ignore model: str, From 325e54390f7129a0f6e2c62c150d2f0b24771a44 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:08:13 +0000 Subject: [PATCH 3/5] style(completion): satisfy the strict ruff gate on the dispatch helpers The dispatch-extraction refactor tripped the strict-rule budget gate. The 61 _complete_ helpers were unannotated (ANN202), the verbatim helper bodies carried redundant assign-then-return locals (RET504), and one List annotation remained (UP006) Annotate every helper with a shared _CompletionDispatchResult alias, the same ModelResponse / CustomStreamWrapper / awaitable contract completion itself returns, inline the redundant locals, and switch the lone List to list. Eleven helpers return a provider SDK type broader than the dispatch contract, so they carry a documented pyright: ignore[reportReturnType] mirroring completion's own pattern; a bare type: ignore does not suppress here because enableTypeIgnoreComments is off. Ratchet the RET504 baseline down to 702 to match the lower count --- litellm/main.py | 345 +++++++++++++++++----------------------- ruff-strict-budget.json | 2 +- 2 files changed, 144 insertions(+), 203 deletions(-) diff --git a/litellm/main.py b/litellm/main.py index 9f69d9c9875c..304b22e85e73 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1104,7 +1104,7 @@ class _CompletionDispatchContext: logging: LiteLLMLoggingObj max_retries: Optional[int] max_tokens: Optional[int] - messages: List + messages: list metadata: Optional[dict] model: str model_response: ModelResponse @@ -1119,7 +1119,14 @@ class _CompletionDispatchContext: top_p: Optional[float] -def _complete_azure(ctx: _CompletionDispatchContext): +_CompletionDispatchResult = Union[ + Coroutine[Any, Any, Union[ModelResponse, CustomStreamWrapper]], + ModelResponse, + CustomStreamWrapper, +] + + +def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: _azure_detection_model = ctx._azure_detection_model acompletion = ctx.acompletion api_base = ctx.api_base @@ -1255,10 +1262,10 @@ def _complete_azure(ctx: _CompletionDispatchContext): }, ) - return response + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_azure_text(ctx: _CompletionDispatchContext): +def _complete_azure_text(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1353,7 +1360,7 @@ def _complete_azure_text(ctx: _CompletionDispatchContext): return response -def _complete_deepseek(ctx: _CompletionDispatchContext): +def _complete_deepseek(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1404,7 +1411,7 @@ def _complete_deepseek(ctx: _CompletionDispatchContext): return response -def _complete_azure_ai(ctx: _CompletionDispatchContext): +def _complete_azure_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1560,7 +1567,9 @@ def _complete_azure_ai(ctx: _CompletionDispatchContext): return response -def _complete_text_completion_openai(ctx: _CompletionDispatchContext): +def _complete_text_completion_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1645,12 +1654,12 @@ def _complete_text_completion_openai(ctx: _CompletionDispatchContext): original_response=_response, additional_args={"headers": headers}, ) - response = _response - - return response + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_fireworks_ai(ctx: _CompletionDispatchContext): +def _complete_fireworks_ai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1701,7 +1710,7 @@ def _complete_fireworks_ai(ctx: _CompletionDispatchContext): return response -def _complete_heroku(ctx: _CompletionDispatchContext): +def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1751,7 +1760,7 @@ def _complete_heroku(ctx: _CompletionDispatchContext): return response -def _complete_ragflow(ctx: _CompletionDispatchContext): +def _complete_ragflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1801,7 +1810,7 @@ def _complete_ragflow(ctx: _CompletionDispatchContext): return response -def _complete_xai(ctx: _CompletionDispatchContext): +def _complete_xai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1852,7 +1861,7 @@ def _complete_xai(ctx: _CompletionDispatchContext): return response -def _complete_groq(ctx: _CompletionDispatchContext): +def _complete_groq(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1894,7 +1903,7 @@ def _complete_groq(ctx: _CompletionDispatchContext): ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in optional_params[k] = v - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -1913,10 +1922,10 @@ def _complete_groq(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_bedrock_mantle(ctx: _CompletionDispatchContext): +def _complete_bedrock_mantle( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -1940,7 +1949,7 @@ def _complete_bedrock_mantle(ctx: _CompletionDispatchContext): for k, v in config.items(): if k not in optional_params: optional_params[k] = v - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -1959,10 +1968,8 @@ def _complete_bedrock_mantle(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_a2a(ctx: _CompletionDispatchContext): +def _complete_a2a(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2004,7 +2011,7 @@ def _complete_a2a(ctx: _CompletionDispatchContext): headers = headers or litellm.headers - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -2024,10 +2031,8 @@ def _complete_a2a(ctx: _CompletionDispatchContext): provider_config=provider_config, ) - return response - -def _complete_gigachat(ctx: _CompletionDispatchContext): +def _complete_gigachat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2089,7 +2094,7 @@ def _complete_gigachat(ctx: _CompletionDispatchContext): return response -def _complete_sap(ctx: _CompletionDispatchContext): +def _complete_sap(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2115,7 +2120,7 @@ def _complete_sap(ctx: _CompletionDispatchContext): ): # completion(top_k=3) > openai_config(top_k=3) <- allows for dynamic variables to be passed in optional_params[k] = v - response = sap_gen_ai_hub_chat_completions.completion( + return sap_gen_ai_hub_chat_completions.completion( model=model, messages=messages, headers=headers, @@ -2134,10 +2139,10 @@ def _complete_sap(ctx: _CompletionDispatchContext): stream=stream, ) - return response - -def _complete_aiohttp_openai(ctx: _CompletionDispatchContext): +def _complete_aiohttp_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2173,7 +2178,7 @@ def _complete_aiohttp_openai(ctx: _CompletionDispatchContext): if extra_headers is not None: optional_params["extra_headers"] = extra_headers - response = base_llm_aiohttp_handler.completion( + return base_llm_aiohttp_handler.completion( model=model, messages=messages, headers=headers, @@ -2191,10 +2196,8 @@ def _complete_aiohttp_openai(ctx: _CompletionDispatchContext): stream=stream, ) - return response - -def _complete_cometapi(ctx: _CompletionDispatchContext): +def _complete_cometapi(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2253,7 +2256,7 @@ def _complete_cometapi(ctx: _CompletionDispatchContext): return response -def _complete_minimax(ctx: _CompletionDispatchContext): +def _complete_minimax(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2304,7 +2307,7 @@ def _complete_minimax(ctx: _CompletionDispatchContext): return response -def _complete_hosted_vllm(ctx: _CompletionDispatchContext): +def _complete_hosted_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2348,7 +2351,9 @@ def _complete_hosted_vllm(ctx: _CompletionDispatchContext): return response -def _complete_custom_openai(ctx: _CompletionDispatchContext): +def _complete_custom_openai( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2493,10 +2498,10 @@ def _complete_custom_openai(ctx: _CompletionDispatchContext): additional_args={"headers": headers}, ) - return response + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_mistral(ctx: _CompletionDispatchContext): +def _complete_mistral(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2522,7 +2527,7 @@ def _complete_mistral(ctx: _CompletionDispatchContext): or "https://api.mistral.ai/v1" ) - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, messages=messages, api_base=api_base, @@ -2542,10 +2547,8 @@ def _complete_mistral(ctx: _CompletionDispatchContext): provider_config=provider_config, ) - return response - -def _complete_replicate(ctx: _CompletionDispatchContext): +def _complete_replicate(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2601,12 +2604,12 @@ def _complete_replicate(ctx: _CompletionDispatchContext): original_response=model_response, ) - response = model_response - - return response + return model_response -def _complete_anthropic_text(ctx: _CompletionDispatchContext): +def _complete_anthropic_text( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2651,7 +2654,7 @@ def _complete_anthropic_text(ctx: _CompletionDispatchContext): "LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix" ) - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -2669,10 +2672,8 @@ def _complete_anthropic_text(ctx: _CompletionDispatchContext): logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) - return response - -def _complete_anthropic(ctx: _CompletionDispatchContext): +def _complete_anthropic(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2746,12 +2747,10 @@ def _complete_anthropic(ctx: _CompletionDispatchContext): api_key=api_key, original_response=response, ) - response = response - return response -def _complete_nlp_cloud(ctx: _CompletionDispatchContext): +def _complete_nlp_cloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2808,12 +2807,10 @@ def _complete_nlp_cloud(ctx: _CompletionDispatchContext): original_response=response, ) - response = response - - return response + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_aleph_alpha(ctx: _CompletionDispatchContext): +def _complete_aleph_alpha(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base api_key = ctx.api_key litellm_params = ctx.litellm_params @@ -2856,19 +2853,16 @@ def _complete_aleph_alpha(ctx: _CompletionDispatchContext): if "stream" in optional_params and optional_params["stream"] is True: # don't try to access stream object, - response = CustomStreamWrapper( + return CustomStreamWrapper( model_response, model, custom_llm_provider="aleph_alpha", logging_obj=logging, ) - return response - response = model_response - - return response + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_cohere_chat(ctx: _CompletionDispatchContext): +def _complete_cohere_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -2923,7 +2917,7 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext): verbose_logger.debug(f"Model: {model}, API Base: {api_base}") verbose_logger.debug(f"Provider Config: {provider_config}") - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -2942,10 +2936,8 @@ def _complete_cohere_chat(ctx: _CompletionDispatchContext): logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) - return response - -def _complete_maritalk(ctx: _CompletionDispatchContext): +def _complete_maritalk(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base api_key = ctx.api_key custom_prompt_dict = ctx.custom_prompt_dict @@ -2971,7 +2963,7 @@ def _complete_maritalk(ctx: _CompletionDispatchContext): or "https://chat.maritaca.ai/api" ) - model_response = openai_like_chat_completion.completion( + return openai_like_chat_completion.completion( model=model, messages=messages, api_base=api_base, @@ -2987,12 +2979,8 @@ def _complete_maritalk(ctx: _CompletionDispatchContext): custom_prompt_dict=custom_prompt_dict, ) - response = model_response - - return response - -def _complete_amazon_nova(ctx: _CompletionDispatchContext): +def _complete_amazon_nova(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base api_key = ctx.api_key custom_llm_provider = ctx.custom_llm_provider @@ -3018,7 +3006,7 @@ def _complete_amazon_nova(ctx: _CompletionDispatchContext): or get_secret_str("AMAZON_NOVA_API_BASE") or "https://api.nova.amazon.com/v1" ) - response = openai_like_chat_completion.completion( + return openai_like_chat_completion.completion( model=model, messages=messages, api_base=api_base, @@ -3035,10 +3023,8 @@ def _complete_amazon_nova(ctx: _CompletionDispatchContext): custom_prompt_dict=custom_prompt_dict, ) - return response - -def _complete_huggingface(ctx: _CompletionDispatchContext): +def _complete_huggingface(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3062,7 +3048,7 @@ def _complete_huggingface(ctx: _CompletionDispatchContext): or litellm.api_key ) hf_headers = headers or litellm.headers - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, messages=messages, headers=hf_headers, @@ -3080,10 +3066,8 @@ def _complete_huggingface(ctx: _CompletionDispatchContext): stream=stream, ) - return response - -def _complete_oci(ctx: _CompletionDispatchContext): +def _complete_oci(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3099,7 +3083,7 @@ def _complete_oci(ctx: _CompletionDispatchContext): stream = ctx.stream timeout = ctx.timeout - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, messages=messages, headers=headers, @@ -3117,10 +3101,8 @@ def _complete_oci(ctx: _CompletionDispatchContext): stream=stream, ) - return response - -def _complete_compactifai(ctx: _CompletionDispatchContext): +def _complete_compactifai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3142,7 +3124,7 @@ def _complete_compactifai(ctx: _CompletionDispatchContext): api_base = api_base or "https://api.compactif.ai/v1" ## COMPLETION CALL - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, messages=messages, headers=headers, @@ -3161,10 +3143,8 @@ def _complete_compactifai(ctx: _CompletionDispatchContext): provider_config=provider_config, ) - return response - -def _complete_oobabooga(ctx: _CompletionDispatchContext): +def _complete_oobabooga(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base litellm_params = ctx.litellm_params logger_fn = ctx.logger_fn @@ -3189,19 +3169,16 @@ def _complete_oobabooga(ctx: _CompletionDispatchContext): ) if "stream" in optional_params and optional_params["stream"] is True: # don't try to access stream object, - response = CustomStreamWrapper( + return CustomStreamWrapper( model_response, model, custom_llm_provider="oobabooga", logging_obj=logging, ) - return response - response = model_response - - return response + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_databricks(ctx: _CompletionDispatchContext): +def _complete_databricks(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3273,7 +3250,7 @@ def _complete_databricks(ctx: _CompletionDispatchContext): return response -def _complete_datarobot(ctx: _CompletionDispatchContext): +def _complete_datarobot(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3290,7 +3267,7 @@ def _complete_datarobot(ctx: _CompletionDispatchContext): stream = ctx.stream timeout = ctx.timeout - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, messages=messages, headers=headers, @@ -3309,10 +3286,8 @@ def _complete_datarobot(ctx: _CompletionDispatchContext): provider_config=provider_config, ) - return response - -def _complete_openrouter(ctx: _CompletionDispatchContext): +def _complete_openrouter(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3396,7 +3371,9 @@ def _complete_openrouter(ctx: _CompletionDispatchContext): return response -def _complete_vercel_ai_gateway(ctx: _CompletionDispatchContext): +def _complete_vercel_ai_gateway( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3474,7 +3451,9 @@ def _complete_vercel_ai_gateway(ctx: _CompletionDispatchContext): return response -def _complete_vertex_ai_beta(ctx: _CompletionDispatchContext): +def _complete_vertex_ai_beta( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3517,7 +3496,7 @@ def _complete_vertex_ai_beta(ctx: _CompletionDispatchContext): api_base = api_base or litellm.api_base or get_secret("GEMINI_API_BASE") new_params = safe_deep_copy(optional_params or {}) - response = vertex_chat_completion.completion( # type: ignore + return vertex_chat_completion.completion( # type: ignore model=model, messages=messages, model_response=model_response, @@ -3539,10 +3518,8 @@ def _complete_vertex_ai_beta(ctx: _CompletionDispatchContext): extra_headers=headers, ) - return response - -def _complete_vertex_ai(ctx: _CompletionDispatchContext): +def _complete_vertex_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base client = ctx.client @@ -3722,19 +3699,16 @@ def _complete_vertex_ai(ctx: _CompletionDispatchContext): and optional_params["stream"] is True and acompletion is False ): - response = CustomStreamWrapper( + return CustomStreamWrapper( model_response, model, custom_llm_provider="vertex_ai", logging_obj=logging, ) - return response - response = model_response - - return response + return model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_predibase(ctx: _CompletionDispatchContext): +def _complete_predibase(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3799,12 +3773,12 @@ def _complete_predibase(ctx: _CompletionDispatchContext): and acompletion is False ): return _model_response - response = _model_response - - return response + return _model_response -def _complete_text_completion_codestral(ctx: _CompletionDispatchContext): +def _complete_text_completion_codestral( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3852,13 +3826,13 @@ def _complete_text_completion_codestral(ctx: _CompletionDispatchContext): and optional_params["stream"] is True and acompletion is False ): - return _model_response - response = _model_response - - return response + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract + return _model_response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_text_completion_inception(ctx: _CompletionDispatchContext): +def _complete_text_completion_inception( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3933,12 +3907,12 @@ def _complete_text_completion_inception(ctx: _CompletionDispatchContext): original_response=_response, additional_args={"headers": headers}, ) - response = _response - - return response + return _response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_sagemaker_chat(ctx: _CompletionDispatchContext): +def _complete_sagemaker_chat( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -3954,7 +3928,7 @@ def _complete_sagemaker_chat(ctx: _CompletionDispatchContext): stream = ctx.stream timeout = ctx.timeout - model_response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -3972,13 +3946,8 @@ def _complete_sagemaker_chat(ctx: _CompletionDispatchContext): client=client, ) - ## RESPONSE OBJECT - response = model_response - - return response - -def _complete_sagemaker(ctx: _CompletionDispatchContext): +def _complete_sagemaker(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion custom_prompt_dict = ctx.custom_prompt_dict hf_model_name = ctx.hf_model_name @@ -3990,7 +3959,7 @@ def _complete_sagemaker(ctx: _CompletionDispatchContext): model_response = ctx.model_response optional_params = ctx.optional_params - model_response = sagemaker_llm.completion( + return sagemaker_llm.completion( model=model, messages=messages, model_response=model_response, @@ -4005,13 +3974,8 @@ def _complete_sagemaker(ctx: _CompletionDispatchContext): acompletion=acompletion, ) - ## RESPONSE OBJECT - response = model_response - - return response - -def _complete_bedrock(ctx: _CompletionDispatchContext): +def _complete_bedrock(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4059,7 +4023,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext): provider=LlmProviders.BEDROCK, ) model = BedrockModelInfo.get_claude_platform_model(model) - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4078,7 +4042,6 @@ def _complete_bedrock(ctx: _CompletionDispatchContext): client=client, provider_config=provider_config, ) - return response elif bedrock_route == "converse": model = model.replace("converse/", "") response = bedrock_converse_chat_completion.completion( @@ -4139,7 +4102,7 @@ def _complete_bedrock(ctx: _CompletionDispatchContext): return response -def _complete_watsonx(ctx: _CompletionDispatchContext): +def _complete_watsonx(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4155,7 +4118,7 @@ def _complete_watsonx(ctx: _CompletionDispatchContext): optional_params = ctx.optional_params timeout = ctx.timeout - response = watsonx_chat_completion.completion( + return watsonx_chat_completion.completion( model=model, messages=messages, headers=headers, @@ -4175,10 +4138,10 @@ def _complete_watsonx(ctx: _CompletionDispatchContext): custom_llm_provider="watsonx", ) - return response - -def _complete_watsonx_text(ctx: _CompletionDispatchContext): +def _complete_watsonx_text( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4235,7 +4198,7 @@ def _complete_watsonx_text(ctx: _CompletionDispatchContext): if token is not None: optional_params["token"] = token - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4254,10 +4217,8 @@ def _complete_watsonx_text(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_vllm(ctx: _CompletionDispatchContext): +def _complete_vllm(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: custom_prompt_dict = ctx.custom_prompt_dict litellm_params = ctx.litellm_params logger_fn = ctx.logger_fn @@ -4283,21 +4244,18 @@ def _complete_vllm(ctx: _CompletionDispatchContext): if "stream" in optional_params and optional_params["stream"] is True: ## [BETA] # don't try to access stream object, - response = CustomStreamWrapper( + return CustomStreamWrapper( model_response, model, custom_llm_provider="vllm", logging_obj=logging, ) - return response ## RESPONSE OBJECT - response = model_response - - return response + return model_response -def _complete_ollama(ctx: _CompletionDispatchContext): +def _complete_ollama(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4322,7 +4280,7 @@ def _complete_ollama(ctx: _CompletionDispatchContext): if api_key is not None and "Authorization" not in headers: headers["Authorization"] = f"Bearer {api_key}" - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4341,10 +4299,8 @@ def _complete_ollama(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_ollama_chat(ctx: _CompletionDispatchContext): +def _complete_ollama_chat(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4376,7 +4332,7 @@ def _complete_ollama_chat(ctx: _CompletionDispatchContext): if api_key is not None and "Authorization" not in headers: headers["Authorization"] = f"Bearer {api_key}" - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4395,10 +4351,8 @@ def _complete_ollama_chat(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_triton(ctx: _CompletionDispatchContext): +def _complete_triton(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4415,7 +4369,7 @@ def _complete_triton(ctx: _CompletionDispatchContext): timeout = ctx.timeout api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4433,10 +4387,8 @@ def _complete_triton(ctx: _CompletionDispatchContext): logging_obj=logging, ) - return response - -def _complete_cloudflare(ctx: _CompletionDispatchContext): +def _complete_cloudflare(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4467,7 +4419,7 @@ def _complete_cloudflare(ctx: _CompletionDispatchContext): ) custom_prompt_dict = custom_prompt_dict or litellm.custom_prompt_dict - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4485,10 +4437,8 @@ def _complete_cloudflare(ctx: _CompletionDispatchContext): logging_obj=logging, # model call logging done inside the class as we make need to modify I/O to fit aleph alpha's requirements ) - return response - -def _complete_petals(ctx: _CompletionDispatchContext): +def _complete_petals(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base client = ctx.client litellm_params = ctx.litellm_params @@ -4519,19 +4469,16 @@ def _complete_petals(ctx: _CompletionDispatchContext): if stream is True: ## [BETA] # Fake streaming for petals resp_string = model_response["choices"][0]["message"]["content"] - response = CustomStreamWrapper( + return CustomStreamWrapper( resp_string, model, custom_llm_provider="petals", logging_obj=logging, ) - return response - response = model_response - - return response + return model_response -def _complete_snowflake(ctx: _CompletionDispatchContext): +def _complete_snowflake(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4584,7 +4531,7 @@ def _complete_snowflake(ctx: _CompletionDispatchContext): return response -def _complete_gradient_ai(ctx: _CompletionDispatchContext): +def _complete_gradient_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4600,7 +4547,7 @@ def _complete_gradient_ai(ctx: _CompletionDispatchContext): timeout = ctx.timeout api_base = litellm.api_base or api_base - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4618,10 +4565,8 @@ def _complete_gradient_ai(ctx: _CompletionDispatchContext): logging_obj=logging, ) - return response - -def _complete_bytez(ctx: _CompletionDispatchContext): +def _complete_bytez(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4668,7 +4613,7 @@ def _complete_bytez(ctx: _CompletionDispatchContext): return response -def _complete_lemonade(ctx: _CompletionDispatchContext): +def _complete_lemonade(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4715,7 +4660,7 @@ def _complete_lemonade(ctx: _CompletionDispatchContext): return response -def _complete_ovhcloud(ctx: _CompletionDispatchContext): +def _complete_ovhcloud(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4769,7 +4714,7 @@ def _complete_ovhcloud(ctx: _CompletionDispatchContext): return response -def _complete_custom(ctx: _CompletionDispatchContext): +def _complete_custom(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: api_base = ctx.api_base headers = ctx.headers kwargs = ctx.kwargs @@ -4838,12 +4783,12 @@ def _complete_custom(ctx: _CompletionDispatchContext): model_response.choices[0].message.content = string_response # type: ignore model_response.created = int(time.time()) model_response.model = model - response = model_response - - return response + return model_response -def _complete_custom_providers(ctx: _CompletionDispatchContext): +def _complete_custom_providers( + ctx: _CompletionDispatchContext, +) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4905,10 +4850,10 @@ def _complete_custom_providers(ctx: _CompletionDispatchContext): logging_obj=logging, ) - return response + return response # pyright: ignore[reportReturnType] # provider SDK return type is broader than the dispatch contract -def _complete_langgraph(ctx: _CompletionDispatchContext): +def _complete_langgraph(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4937,7 +4882,7 @@ def _complete_langgraph(ctx: _CompletionDispatchContext): headers = headers or litellm.headers - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -4956,10 +4901,8 @@ def _complete_langgraph(ctx: _CompletionDispatchContext): client=client, ) - return response - -def _complete_langflow(ctx: _CompletionDispatchContext): +def _complete_langflow(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: acompletion = ctx.acompletion api_base = ctx.api_base api_key = ctx.api_key @@ -4988,7 +4931,7 @@ def _complete_langflow(ctx: _CompletionDispatchContext): headers = headers or litellm.headers - response = base_llm_http_handler.completion( + return base_llm_http_handler.completion( model=model, stream=stream, messages=messages, @@ -5007,8 +4950,6 @@ def _complete_langflow(ctx: _CompletionDispatchContext): client=client, ) - return response - @tracer.wrap() @client diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json index 62ebdb559fc5..ae46f020de1d 100644 --- a/ruff-strict-budget.json +++ b/ruff-strict-budget.json @@ -300,7 +300,7 @@ "slack": 3 }, "RET504": { - "baseline": 709, + "baseline": 702, "slack": 20 }, "RUF010": { From 7294bc111187bb04204ca6e8e0177387915fcec1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:18:33 +0000 Subject: [PATCH 4/5] refactor(completion): move dispatch types into litellm/types/main.py Relocate the private _CompletionDispatchContext and _CompletionDispatchResult out of main.py into a dedicated litellm/types/main.py so the entrypoint file is not carrying the type classes inline, per review feedback. The definitions are moved verbatim, so the dispatch helpers and completion() are unchanged and the basedpyright/ruff budgets stay identical (the same annotations now live in the new module). Adds a mapped test pinning the frozen+slots invariant the dispatch shape relies on. --- litellm/main.py | 46 ++------------------ litellm/types/main.py | 54 ++++++++++++++++++++++++ tests/test_litellm/types/test_main.py | 61 +++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 42 deletions(-) create mode 100644 litellm/types/main.py create mode 100644 tests/test_litellm/types/test_main.py diff --git a/litellm/main.py b/litellm/main.py index 304b22e85e73..83d16643d853 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -22,7 +22,6 @@ from concurrent import futures from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait from copy import deepcopy -from dataclasses import dataclass from functools import partial from typing import ( TYPE_CHECKING, @@ -119,6 +118,10 @@ ) from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.types.main import ( + _CompletionDispatchContext, + _CompletionDispatchResult, +) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( CustomPricingLiteLLMParams, @@ -1085,47 +1088,6 @@ def _build_custom_pricing_entry( return entry -@dataclass(frozen=True, slots=True) -class _CompletionDispatchContext: - _azure_detection_model: str - acompletion: bool - api_base: Optional[str] - api_key: Optional[str] - api_version: Optional[str] - client: Any - custom_llm_provider: str - custom_prompt_dict: dict - extra_headers: Optional[dict] - headers: dict - hf_model_name: Optional[str] - kwargs: dict - litellm_params: dict - logger_fn: Optional[Callable] - logging: LiteLLMLoggingObj - max_retries: Optional[int] - max_tokens: Optional[int] - messages: list - metadata: Optional[dict] - model: str - model_response: ModelResponse - optional_params: dict - organization: Optional[str] - provider_config: Optional[BaseConfig] - shared_session: Optional["ClientSession"] - stream: Optional[bool] - temperature: Optional[float] - text_completion: bool - timeout: Optional[Union[float, str, httpx.Timeout]] - top_p: Optional[float] - - -_CompletionDispatchResult = Union[ - Coroutine[Any, Any, Union[ModelResponse, CustomStreamWrapper]], - ModelResponse, - CustomStreamWrapper, -] - - def _complete_azure(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult: _azure_detection_model = ctx._azure_detection_model acompletion = ctx.acompletion diff --git a/litellm/types/main.py b/litellm/types/main.py new file mode 100644 index 000000000000..2636a1db5438 --- /dev/null +++ b/litellm/types/main.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, Union + +from litellm.utils import CustomStreamWrapper, ModelResponse + +if TYPE_CHECKING: + import httpx + from aiohttp import ClientSession + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm import BaseConfig + + +@dataclass(frozen=True, slots=True) +class _CompletionDispatchContext: + _azure_detection_model: str + acompletion: bool + api_base: Optional[str] + api_key: Optional[str] + api_version: Optional[str] + client: Any + custom_llm_provider: str + custom_prompt_dict: dict + extra_headers: Optional[dict] + headers: dict + hf_model_name: Optional[str] + kwargs: dict + litellm_params: dict + logger_fn: Optional[Callable] + logging: LiteLLMLoggingObj + max_retries: Optional[int] + max_tokens: Optional[int] + messages: list + metadata: Optional[dict] + model: str + model_response: ModelResponse + optional_params: dict + organization: Optional[str] + provider_config: Optional[BaseConfig] + shared_session: Optional[ClientSession] + stream: Optional[bool] + temperature: Optional[float] + text_completion: bool + timeout: Optional[Union[float, str, httpx.Timeout]] + top_p: Optional[float] + + +_CompletionDispatchResult = Union[ + Coroutine[Any, Any, Union[ModelResponse, CustomStreamWrapper]], + ModelResponse, + CustomStreamWrapper, +] diff --git a/tests/test_litellm/types/test_main.py b/tests/test_litellm/types/test_main.py new file mode 100644 index 000000000000..a6c0cc9fa34a --- /dev/null +++ b/tests/test_litellm/types/test_main.py @@ -0,0 +1,61 @@ +import dataclasses +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../..")) + +from litellm.types.main import _CompletionDispatchContext + + +def _build_context() -> _CompletionDispatchContext: + return _CompletionDispatchContext( + _azure_detection_model="gpt-4o", + acompletion=False, + api_base=None, + api_key=None, + api_version=None, + client=None, + custom_llm_provider="openai", + custom_prompt_dict={}, + extra_headers=None, + headers={}, + hf_model_name=None, + kwargs={}, + litellm_params={}, + logger_fn=None, + logging=None, # type: ignore[arg-type] + max_retries=None, + max_tokens=None, + messages=[], + metadata=None, + model="gpt-4o", + model_response=None, # type: ignore[arg-type] + optional_params={}, + organization=None, + provider_config=None, + shared_session=None, + stream=None, + temperature=None, + text_completion=False, + timeout=None, + top_p=None, + ) + + +def test_dispatch_context_is_frozen(): + """A helper must not be able to re-route the call by rebinding a dispatch + input mid-flight; this pins the frozen invariant the dispatch shape relies on.""" + ctx = _build_context() + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.model = "claude-haiku-4-5" # type: ignore[misc] + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.custom_llm_provider = "anthropic" # type: ignore[misc] + + +def test_dispatch_context_uses_slots(): + """slots=True keeps the per-call context lightweight (no per-instance __dict__).""" + ctx = _build_context() + assert not hasattr(ctx, "__dict__") + assert hasattr(type(ctx), "__slots__") From d6b1dca2b12a2542e3f544781908c247fb8fb34c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 23 Jun 2026 07:13:03 +0000 Subject: [PATCH 5/5] refactor(completion): move dispatch types into litellm/types/completion.py Co-locate the completion dispatch context and result alias with the existing completion request types in litellm/types/completion.py rather than a standalone module. The litellm.utils types they reference are imported under TYPE_CHECKING and the result alias is built from forward references, so this early-imported types module stays free of a circular import. --- litellm/main.py | 2 +- litellm/types/completion.py | 63 ++++++++++++++++++++- litellm/types/main.py | 54 ------------------ tests/test_litellm/types/test_completion.py | 61 +++++++++++++++++++- tests/test_litellm/types/test_main.py | 61 -------------------- 5 files changed, 123 insertions(+), 118 deletions(-) delete mode 100644 litellm/types/main.py delete mode 100644 tests/test_litellm/types/test_main.py diff --git a/litellm/main.py b/litellm/main.py index 83d16643d853..1d75766c7e6b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -118,7 +118,7 @@ ) from litellm.realtime_api.main import _realtime_health_check from litellm.secret_managers.main import get_secret_bool, get_secret_str -from litellm.types.main import ( +from litellm.types.completion import ( _CompletionDispatchContext, _CompletionDispatchResult, ) diff --git a/litellm/types/completion.py b/litellm/types/completion.py index cb263914be82..a91f6234fada 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -1,8 +1,28 @@ -from typing import Iterable, List, Optional, Union +from __future__ import annotations + +from dataclasses import dataclass +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Coroutine, + Iterable, + List, + Optional, + Union, +) from pydantic import BaseModel, ConfigDict from typing_extensions import Literal, Required, TypedDict +if TYPE_CHECKING: + import httpx + from aiohttp import ClientSession + + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.llms.base_llm import BaseConfig + from litellm.utils import CustomStreamWrapper, ModelResponse + class ChatCompletionSystemMessageParam(TypedDict, total=False): content: Required[str] @@ -191,3 +211,44 @@ class CompletionRequest(BaseModel): model_list: Optional[List[str]] = None model_config = ConfigDict(protected_namespaces=(), extra="allow") + + +@dataclass(frozen=True, slots=True) +class _CompletionDispatchContext: + _azure_detection_model: str + acompletion: bool + api_base: Optional[str] + api_key: Optional[str] + api_version: Optional[str] + client: Any + custom_llm_provider: str + custom_prompt_dict: dict + extra_headers: Optional[dict] + headers: dict + hf_model_name: Optional[str] + kwargs: dict + litellm_params: dict + logger_fn: Optional[Callable] + logging: LiteLLMLoggingObj + max_retries: Optional[int] + max_tokens: Optional[int] + messages: list + metadata: Optional[dict] + model: str + model_response: ModelResponse + optional_params: dict + organization: Optional[str] + provider_config: Optional[BaseConfig] + shared_session: Optional[ClientSession] + stream: Optional[bool] + temperature: Optional[float] + text_completion: bool + timeout: Optional[Union[float, str, httpx.Timeout]] + top_p: Optional[float] + + +_CompletionDispatchResult = Union[ + Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]], + "ModelResponse", + "CustomStreamWrapper", +] diff --git a/litellm/types/main.py b/litellm/types/main.py deleted file mode 100644 index 2636a1db5438..000000000000 --- a/litellm/types/main.py +++ /dev/null @@ -1,54 +0,0 @@ -from __future__ import annotations - -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, Union - -from litellm.utils import CustomStreamWrapper, ModelResponse - -if TYPE_CHECKING: - import httpx - from aiohttp import ClientSession - - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj - from litellm.llms.base_llm import BaseConfig - - -@dataclass(frozen=True, slots=True) -class _CompletionDispatchContext: - _azure_detection_model: str - acompletion: bool - api_base: Optional[str] - api_key: Optional[str] - api_version: Optional[str] - client: Any - custom_llm_provider: str - custom_prompt_dict: dict - extra_headers: Optional[dict] - headers: dict - hf_model_name: Optional[str] - kwargs: dict - litellm_params: dict - logger_fn: Optional[Callable] - logging: LiteLLMLoggingObj - max_retries: Optional[int] - max_tokens: Optional[int] - messages: list - metadata: Optional[dict] - model: str - model_response: ModelResponse - optional_params: dict - organization: Optional[str] - provider_config: Optional[BaseConfig] - shared_session: Optional[ClientSession] - stream: Optional[bool] - temperature: Optional[float] - text_completion: bool - timeout: Optional[Union[float, str, httpx.Timeout]] - top_p: Optional[float] - - -_CompletionDispatchResult = Union[ - Coroutine[Any, Any, Union[ModelResponse, CustomStreamWrapper]], - ModelResponse, - CustomStreamWrapper, -] diff --git a/tests/test_litellm/types/test_completion.py b/tests/test_litellm/types/test_completion.py index f24b00df3fcc..cd51913c5ddb 100644 --- a/tests/test_litellm/types/test_completion.py +++ b/tests/test_litellm/types/test_completion.py @@ -8,9 +8,16 @@ pytest tests/test_litellm/types/test_completion.py -v """ +import dataclasses from typing import List -from litellm.types.completion import CompletionRequest, ChatCompletionMessageParam +import pytest + +from litellm.types.completion import ( + ChatCompletionMessageParam, + CompletionRequest, + _CompletionDispatchContext, +) def test_completion_request_messages_type_validation(): @@ -146,3 +153,55 @@ def test_completion_request_with_all_params(): assert request.presence_penalty == 0.0 assert request.stream is False assert request.n == 1 + + +def _build_dispatch_context() -> _CompletionDispatchContext: + return _CompletionDispatchContext( + _azure_detection_model="gpt-4o", + acompletion=False, + api_base=None, + api_key=None, + api_version=None, + client=None, + custom_llm_provider="openai", + custom_prompt_dict={}, + extra_headers=None, + headers={}, + hf_model_name=None, + kwargs={}, + litellm_params={}, + logger_fn=None, + logging=None, # type: ignore[arg-type] + max_retries=None, + max_tokens=None, + messages=[], + metadata=None, + model="gpt-4o", + model_response=None, # type: ignore[arg-type] + optional_params={}, + organization=None, + provider_config=None, + shared_session=None, + stream=None, + temperature=None, + text_completion=False, + timeout=None, + top_p=None, + ) + + +def test_dispatch_context_is_frozen(): + """A helper must not be able to re-route the call by rebinding a dispatch + input mid-flight; this pins the frozen invariant the dispatch shape relies on.""" + ctx = _build_dispatch_context() + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.model = "claude-haiku-4-5" # type: ignore[misc] + with pytest.raises(dataclasses.FrozenInstanceError): + ctx.custom_llm_provider = "anthropic" # type: ignore[misc] + + +def test_dispatch_context_uses_slots(): + """slots=True keeps the per-call context lightweight (no per-instance __dict__).""" + ctx = _build_dispatch_context() + assert not hasattr(ctx, "__dict__") + assert hasattr(type(ctx), "__slots__") diff --git a/tests/test_litellm/types/test_main.py b/tests/test_litellm/types/test_main.py deleted file mode 100644 index a6c0cc9fa34a..000000000000 --- a/tests/test_litellm/types/test_main.py +++ /dev/null @@ -1,61 +0,0 @@ -import dataclasses -import os -import sys - -import pytest - -sys.path.insert(0, os.path.abspath("../..")) - -from litellm.types.main import _CompletionDispatchContext - - -def _build_context() -> _CompletionDispatchContext: - return _CompletionDispatchContext( - _azure_detection_model="gpt-4o", - acompletion=False, - api_base=None, - api_key=None, - api_version=None, - client=None, - custom_llm_provider="openai", - custom_prompt_dict={}, - extra_headers=None, - headers={}, - hf_model_name=None, - kwargs={}, - litellm_params={}, - logger_fn=None, - logging=None, # type: ignore[arg-type] - max_retries=None, - max_tokens=None, - messages=[], - metadata=None, - model="gpt-4o", - model_response=None, # type: ignore[arg-type] - optional_params={}, - organization=None, - provider_config=None, - shared_session=None, - stream=None, - temperature=None, - text_completion=False, - timeout=None, - top_p=None, - ) - - -def test_dispatch_context_is_frozen(): - """A helper must not be able to re-route the call by rebinding a dispatch - input mid-flight; this pins the frozen invariant the dispatch shape relies on.""" - ctx = _build_context() - with pytest.raises(dataclasses.FrozenInstanceError): - ctx.model = "claude-haiku-4-5" # type: ignore[misc] - with pytest.raises(dataclasses.FrozenInstanceError): - ctx.custom_llm_provider = "anthropic" # type: ignore[misc] - - -def test_dispatch_context_uses_slots(): - """slots=True keeps the per-call context lightweight (no per-instance __dict__).""" - ctx = _build_context() - assert not hasattr(ctx, "__dict__") - assert hasattr(type(ctx), "__slots__")