diff --git a/docs/my-website/docs/providers/neosantara.md b/docs/my-website/docs/providers/neosantara.md new file mode 100644 index 000000000000..dcd6114d4bcb --- /dev/null +++ b/docs/my-website/docs/providers/neosantara.md @@ -0,0 +1,201 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Neosantara + +## Overview + +| Property | Details | +|-------|-------| +| Description | Neosantara is a unified LLM gateway designed for developers in Indonesia, providing a single OpenAI-compatible interface to multiple top-tier AI models (OpenAI, Anthropic, Gemini, etc.). | +| Provider Route on LiteLLM | `neosantara/` | +| Link to Provider Doc | [Neosantara Dashboard ↗](https://app.neosantara.xyz) | +| Base URL | `https://api.neosantara.xyz/v1` | +| Supported Operations | [`/chat/completions`](#sample-usage), [`/embeddings`](#embeddings) | + +
+ +## What is Neosantara? + +Neosantara is a unified gateway that lets developers: +- **Access Multiple LLM Providers**: Unified interface for OpenAI, Anthropic, Gemini, and more. +- **Optimized for Indonesia**: Designed specifically for the needs of developers in the region. +- **Unified Billing**: Pay-As-You-Go system with local payment support. +- **OpenAI Compatible**: Seamlessly drop into existing OpenAI-based workflows. + +## Required Variables + +```python showLineNumbers title="Environment Variables" +os.environ["NEOSANTARA_API_KEY"] = "your-neosantara-api-key" +``` + +Get your Neosantara API key from [app.neosantara.xyz](https://app.neosantara.xyz). + +## Usage - LiteLLM Python SDK + + + + +```python showLineNumbers title="Neosantara Non-streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["NEOSANTARA_API_KEY"] = "your-neosantara-api-key" + +messages = [{"content": "What is the capital of Indonesia?", "role": "user"}] + +# Neosantara call +response = completion( + model="neosantara/claude-3-haiku", + messages=messages +) + +print(response) +``` + + + + +```python showLineNumbers title="Neosantara Streaming Completion" +import os +import litellm +from litellm import completion + +os.environ["NEOSANTARA_API_KEY"] = "your-neosantara-api-key" + +messages = [{"content": "Write a short poem about Jakarta", "role": "user"}] + +# Neosantara call with streaming +response = completion( + model="neosantara/claude-3-haiku", + messages=messages, + stream=True +) + +for chunk in response: + print(chunk) +``` + + + + +```python showLineNumbers title="Neosantara Embeddings" +import os +import litellm +from litellm import embedding + +os.environ["NEOSANTARA_API_KEY"] = "your-neosantara-api-key" + +# Neosantara call +response = embedding( + model="neosantara/nusa-embedding-0001", + input=["Hello, how are you?"] +) + +print(response) +``` + + + + +## Usage - LiteLLM Proxy Server + +### 1. Set Neosantara Models on `config.yaml` + +```yaml +model_list: + - model_name: neosantara-claude-3-haiku + litellm_params: + model: neosantara/claude-3-haiku + api_key: os.environ/NEOSANTARA_API_KEY +``` + +### 2. Start Proxy + +```bash +litellm --config config.yaml +``` + +### 3. Test it + + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer sk-1234' \ +--data ' { + "model": "neosantara-claude-3-haiku", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="neosantara-claude-3-haiku", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ] +) + +print(response) +``` + + + +## Supported Models + +We support a wide range of models optimized for the Indonesian context and high-performance tasks. + +| Model Name | Model ID (for LiteLLM) | Provider | Description | +|------------|------------------------|----------|-------------| +| **Nusantara Base** | `neosantara/nusantara-base` | Gemini | Flagship balanced model | +| **Archipelago 70B** | `neosantara/archipelago-70b` | Llama 3.3 | Cultural context awareness | +| **Garda Beta Mini** | `neosantara/garda-beta-mini` | Groq/Paxsenix | Fast & efficient Indonesian understanding | +| **Claude 3 Haiku** | `neosantara/claude-3-haiku` | Bedrock | Near-instant responsiveness | +| **Claude 3 Sonnet** | `neosantara/claude-3-sonnet` | Bedrock | Balance of intelligence and speed | +| **Sahabat AI Llama v4** | `neosantara/sahabat-ai-llama-v4` | SahabatAI | Fine-tuned for Sahabat AI ecosystem | +| **Nusa Embedding 0001**| `neosantara/nusa-embedding-0001` | Embedding | Optimized for Indonesian search | + +:::info +**Note:** You can use any model supported by Neosantara by adding the `neosantara/` prefix to the model name in your LiteLLM calls. +::: + +## Supported OpenAI Parameters + +Neosantara supports all standard OpenAI-compatible parameters: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `messages` | array | **Required**. Array of message objects with 'role' and 'content' | +| `model` | string | **Required**. Model ID (e.g., `claude-3-haiku`, `archipelago-70b`) | +| `stream` | boolean | Optional. Enable streaming responses | +| `temperature` | float | Optional. Sampling temperature | +| `top_p` | float | Optional. Nucleus sampling parameter | +| `max_tokens` | integer | Optional. Maximum tokens to generate | +| `tools` | array | Optional. List of available tools/functions | +| `tool_choice` | string/object | Optional. Control tool/function calling | + +## Additional Resources + +- [Neosantara Dashboard](https://app.neosantara.xyz) +- [API Documentation](https://docs.neosantara.xyz) \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 033faeda7c9b..809f469eb5f3 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -790,6 +790,7 @@ const sidebars = { "providers/moonshot", "providers/morph", "providers/nebius", + "providers/neosantara", "providers/nlp_cloud", "providers/nano-gpt", "providers/novita", diff --git a/litellm/__init__.py b/litellm/__init__.py index f857e10eed33..39daa9fed39d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -543,6 +543,7 @@ def identify(event_details): nscale_models: Set = set() nebius_models: Set = set() nebius_embedding_models: Set = set() +neosantara_embedding_models: Set = set() aiml_models: Set = set() deepgram_models: Set = set() elevenlabs_models: Set = set() @@ -571,6 +572,7 @@ def identify(event_details): aws_polly_models: Set = set() gigachat_models: Set = set() llamagate_models: Set = set() +neosantara_models: Set = set() def is_bedrock_pricing_only_model(key: str) -> bool: @@ -832,6 +834,11 @@ def add_known_models(): gigachat_models.add(key) elif value.get("litellm_provider") == "llamagate": llamagate_models.add(key) + elif value.get("litellm_provider") == "neosantara": + if value.get("mode") == "embedding": + neosantara_embedding_models.add(key) + else: + neosantara_models.add(key) add_known_models() @@ -1042,6 +1049,7 @@ def add_known_models(): "aws_polly": aws_polly_models, "gigachat": gigachat_models, "llamagate": llamagate_models, + "neosantara": neosantara_models | neosantara_embedding_models, } # mapping for those models which have larger equivalents @@ -1077,6 +1085,7 @@ def add_known_models(): | nebius_embedding_models | sambanova_embedding_models | ovhcloud_embedding_models + | neosantara_embedding_models ) ####### IMAGE GENERATION MODELS ################### diff --git a/litellm/constants.py b/litellm/constants.py index 444e78f8ed42..14c0fa97292f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -455,6 +455,7 @@ "lemonade", "docker_model_runner", "amazon_nova", + "neosantara", ] LITELLM_EMBEDDING_PROVIDERS_SUPPORTING_INPUT_ARRAY_OF_TOKENS = [ @@ -462,6 +463,7 @@ "azure", "hosted_vllm", "nebius", + "neosantara", ] @@ -664,6 +666,7 @@ "clarifai", "docker_model_runner", "ragflow", + "neosantara", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` @@ -904,6 +907,12 @@ ] ) +neosantara_embedding_models: set = set( + [ + "nusa-embedding-0001", + ] +) + WANDB_MODELS: set = set( [ # openai models diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 718773a1b162..dccd305f9018 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -195,6 +195,21 @@ def get_llm_provider( # noqa: PLR0915 return model, custom_llm_provider, dynamic_api_key, api_base # check if api base is a known openai compatible endpoint if api_base: + if "api.neosantara.xyz/v1" in api_base: + custom_llm_provider = "neosantara" + dynamic_api_key = get_secret_str("NEOSANTARA_API_KEY") + if api_base is not None and not isinstance(api_base, str): + raise Exception( + "api base needs to be a string. api_base={}".format(api_base) + ) + if dynamic_api_key is not None and not isinstance(dynamic_api_key, str): + raise Exception( + "dynamic_api_key needs to be a string. dynamic_api_key={}".format( + dynamic_api_key + ) + ) + return model, custom_llm_provider, dynamic_api_key, api_base + for endpoint in litellm.openai_compatible_endpoints: if endpoint in api_base: if endpoint == "api.perplexity.ai": diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b4f9cbe42de1..6ce14919674e 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -86,5 +86,12 @@ "headers": { "api-subscription-key": "{api_key}" } + }, + "neosantara": { + "base_url": "https://api.neosantara.xyz/v1", + "api_key_env": "NEOSANTARA_API_KEY", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } -} +} \ No newline at end of file diff --git a/litellm/main.py b/litellm/main.py index 6d6bef81c266..5644dab23e56 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4756,6 +4756,8 @@ def embedding( # noqa: PLR0915 or custom_llm_provider == "nvidia_nim" or custom_llm_provider == "litellm_proxy" or (model in litellm.open_ai_embedding_models and custom_llm_provider is None) + or custom_llm_provider in litellm.openai_compatible_providers + or JSONProviderRegistry.exists(custom_llm_provider) ): api_base = ( api_base diff --git a/litellm/types/utils.py b/litellm/types/utils.py index e1f780ffcc38..d18c444cc182 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3084,6 +3084,7 @@ class LlmProviders(str, Enum): GRADIENT_AI = "gradient_ai" LLAMA = "meta_llama" NSCALE = "nscale" + NEOSANTARA = "neosantara" PG_VECTOR = "pg_vector" S3_VECTORS = "s3_vectors" HELICONE = "helicone" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 9fa68a516735..991bee82b69a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -29912,7 +29912,9 @@ "mode": "chat", "output_cost_per_token": 1e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_tool_choice": true }, @@ -29925,7 +29927,9 @@ "mode": "chat", "output_cost_per_token": 4e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_tool_choice": true }, @@ -29938,7 +29942,9 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_tool_choice": true }, @@ -29951,7 +29957,9 @@ "mode": "chat", "output_cost_per_token": 1.2e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing", - "supported_regions": ["global"], + "supported_regions": [ + "global" + ], "supports_function_calling": true, "supports_tool_choice": true }, @@ -34899,5 +34907,339 @@ "mode": "chat", "output_cost_per_token": 0, "supports_reasoning": true + }, + "neosantara/nusantara-base": { + "max_input_tokens": 64000, + "max_output_tokens": 2048, + "input_cost_per_token": 1.875e-08, + "output_cost_per_token": 9.375e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/archipelago-70b": { + "max_input_tokens": 24000, + "max_output_tokens": 2048, + "input_cost_per_token": 2.94375e-07, + "output_cost_per_token": 2.2839375e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/garda-beta-mini": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 1.5625e-07, + "output_cost_per_token": 6.561875e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/sea-lion-v4-27b-it": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 2.1875e-08, + "output_cost_per_token": 3.5e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/claude-3-haiku": { + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/claude-3-sonnet": { + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gemma-3-27b-it": { + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "input_cost_per_token": 1.25e-08, + "output_cost_per_token": 9.375e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/granite-3-8b-instruct": { + "max_input_tokens": 128000, + "max_output_tokens": 2048, + "input_cost_per_token": 1.8e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/llama-3.3-70b-instruct": { + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "input_cost_per_token": 2.8125e-07, + "output_cost_per_token": 2e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/holo2-30b": { + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "input_cost_per_token": 9.375e-08, + "output_cost_per_token": 2.8125e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/hermes-2-pro-mistral-7b": { + "max_input_tokens": 24000, + "max_output_tokens": 4096, + "input_cost_per_token": 2.8125e-08, + "output_cost_per_token": 7.5e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gemma2-9b-it": { + "max_input_tokens": 8192, + "max_output_tokens": 8192, + "input_cost_per_token": 1.25e-08, + "output_cost_per_token": 6.25e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/qwen3-32b": { + "max_input_tokens": 131072, + "max_output_tokens": 40960, + "input_cost_per_token": 3.0175e-07, + "output_cost_per_token": 6.139375e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/llama-3.3-70b-turbo": { + "max_input_tokens": 8192, + "max_output_tokens": 4096, + "input_cost_per_token": 1.875e-08, + "output_cost_per_token": 7.5e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gpt-oss-20b": { + "max_input_tokens": 131072, + "max_output_tokens": 8192, + "input_cost_per_token": 2.5e-08, + "output_cost_per_token": 1e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/llama-3.2-11b": { + "max_input_tokens": 8192, + "max_output_tokens": 2048, + "input_cost_per_token": 6.25e-09, + "output_cost_per_token": 2.5e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/apriel-1.6-thinker": { + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "input_cost_per_token": 1.875e-07, + "output_cost_per_token": 5.625e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2": { + "max_input_tokens": 131000, + "max_output_tokens": 128000, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/deepseek-chat-v3.1": { + "max_input_tokens": 164000, + "max_output_tokens": 4029, + "input_cost_per_token": 2.083125e-07, + "output_cost_per_token": 5.208125e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/glm-4.6": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gemini-3-pro-preview": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 2.5e-06, + "output_cost_per_token": 7.5e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gemini-3-flash-preview": { + "max_input_tokens": 1000000, + "max_output_tokens": 8192, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 4e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/claude-4.5-sonnet": { + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/gemini-3-flash": { + "max_input_tokens": 1000000, + "max_output_tokens": 2400, + "input_cost_per_token": 5e-07, + "output_cost_per_token": 3e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/grok-code-fast": { + "max_input_tokens": 256000, + "max_output_tokens": 4096, + "input_cost_per_token": 2e-07, + "output_cost_per_token": 1.5e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/grok-4.1-fast-non-reasoning": { + "max_input_tokens": 2000000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-07, + "output_cost_per_token": 7.2e-07, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/nusa-embedding-0001": { + "max_input_tokens": 8192, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0, + "litellm_provider": "neosantara", + "mode": "embedding" + }, + "neosantara/gemini-embedding-001": { + "max_input_tokens": 2048, + "input_cost_per_token": 1e-07, + "output_cost_per_token": 0, + "litellm_provider": "neosantara", + "mode": "embedding" + }, + "neosantara/llama-3.3-nemotron-super-49b-v1.5": { + "max_input_tokens": 132000, + "max_output_tokens": 128000, + "input_cost_per_token": 2.85e-07, + "output_cost_per_token": 1.9994375e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2:latest": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2:search": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2:research": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k1": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2:math": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2:silent": { + "max_input_tokens": 128000, + "max_output_tokens": 4096, + "input_cost_per_token": 1.040125e-06, + "output_cost_per_token": 3.1205625e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/glm-4.6-plus": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/glm-4.6v-flash": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.5625e-08, + "output_cost_per_token": 4.6875e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/glm-4.5-flash": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.5625e-08, + "output_cost_per_token": 4.6875e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/glm-4.7": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 6.25e-07, + "output_cost_per_token": 1.875e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/kimi-k2-thinking": { + "max_input_tokens": 128000, + "max_output_tokens": 8192, + "input_cost_per_token": 1.25e-06, + "output_cost_per_token": 3.75e-06, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/llama-guard-3-8b": { + "max_input_tokens": 131072, + "max_output_tokens": 256, + "input_cost_per_token": 4.8e-07, + "output_cost_per_token": 3e-08, + "litellm_provider": "neosantara", + "mode": "chat" + }, + "neosantara/sahabat-ai-llama-v4": { + "max_input_tokens": 32768, + "max_output_tokens": 4096, + "input_cost_per_token": 9.375e-09, + "output_cost_per_token": 3.75e-08, + "litellm_provider": "neosantara", + "mode": "chat" } } \ No newline at end of file diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index fd17b5309e83..614cf6ee136a 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1467,6 +1467,37 @@ "interactions": true } }, + "neosantara": { + "display_name": "Neosantara (`neosantara`)", + "url": "https://docs.litellm.ai/docs/providers/neosantara", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "container": true, + "compact": true, + "a2a": true, + "interactions": true, + "vector_store_files": true, + "vector_stores_create": true, + "vector_stores_search": true, + "assistants": true, + "container_files": true, + "fine_tuning": true, + "image_variations": true, + "rag_ingest": true, + "rag_query": true, + "realtime": true, + "text_completion": true + } + }, "nlp_cloud": { "display_name": "NLP Cloud (`nlp_cloud`)", "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", diff --git a/tests/test_litellm/test_neosantara.py b/tests/test_litellm/test_neosantara.py new file mode 100644 index 000000000000..08d7c04c26a4 --- /dev/null +++ b/tests/test_litellm/test_neosantara.py @@ -0,0 +1,241 @@ +import os +import pytest +import litellm +import httpx +import json +from unittest.mock import MagicMock, patch +from openai.types.chat import ChatCompletion +from openai.types.chat.chat_completion import Choice +from openai.types.chat.chat_completion_message import ChatCompletionMessage +from openai.types.completion_usage import CompletionUsage +from openai.types.create_embedding_response import CreateEmbeddingResponse +from openai.types.embedding import Embedding + +# This class mimics the structure of the `APIResponse` object returned by `openai`'s `with_raw_response` +class MockAPIResponse: + def __init__(self, mock_obj): + self._mock_obj = mock_obj + self.headers = {"x-request-id": "some-id"} + + def parse(self): + return self._mock_obj + +def test_neosantara_provider_info(): + """ + Test that Neosantara is correctly recognized as a provider and maps to the correct base URL. + """ + model, provider, api_key, api_base = litellm.get_llm_provider("neosantara/claude-3-haiku") + + assert provider == "neosantara" + assert api_base == "https://api.neosantara.xyz/v1" + assert model == "claude-3-haiku" + +def test_neosantara_auto_detection_api_base(): + """ + Test that Neosantara is automatically detected when the Neosantara API base URL is provided. + """ + model, provider, api_key, api_base = litellm.get_llm_provider( + model="claude-3-haiku", + api_base="https://api.neosantara.xyz/v1" + ) + + assert provider == "neosantara" + assert api_base == "https://api.neosantara.xyz/v1" + assert model == "claude-3-haiku" + +def test_neosantara_auto_detection_api_base_no_protocol(): + """ + Test that Neosantara is automatically detected when the Neosantara API base URL is provided without protocol. + """ + model, provider, api_key, api_base = litellm.get_llm_provider( + model="claude-3-haiku", + api_base="api.neosantara.xyz/v1" + ) + + assert provider == "neosantara" + assert api_base == "api.neosantara.xyz/v1" + assert model == "claude-3-haiku" + +@patch("openai.resources.chat.completions.Completions.create") +def test_neosantara_completion_formatting(mock_create): + """ + Test that a completion call to Neosantara formats the request correctly as an OpenAI-like call. + """ + os.environ["NEOSANTARA_API_KEY"] = "sk-1234" + + # Define the mock response object to be returned by the patched method + mock_choice = Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + content="Hello world", + role="assistant", + function_call=None, + tool_calls=None, + ) + ) + mock_usage = CompletionUsage( + completion_tokens=10, + prompt_tokens=10, + total_tokens=20 + ) + mock_chat_completion = ChatCompletion( + id="chatcmpl-123", + choices=[mock_choice], + created=1677610602, + model="claude-3-haiku", + object="chat.completion", + usage=mock_usage, + system_fingerprint=None + ) + + mock_create.return_value = MockAPIResponse(mock_chat_completion) + + response = litellm.completion( + model="neosantara/claude-3-haiku", + messages=[{"role": "user", "content": "hi"}], + api_key="sk-1234" + ) + + # Assert that the mocked 'create' method was called with the correct parameters + mock_create.assert_called_once() + called_args, called_kwargs = mock_create.call_args + assert called_kwargs["model"] == "claude-3-haiku" + assert called_kwargs["messages"] == [{"role": "user", "content": "hi"}] + + # Assertions on the transformed response + assert response.choices[0].message.content == "Hello world" + assert response._hidden_params["custom_llm_provider"] == "neosantara" + assert response._hidden_params["api_base"] == "https://api.neosantara.xyz/v1" +@patch("openai.resources.embeddings.Embeddings.create") +def test_neosantara_embedding_formatting(mock_create): + """ + Test that an embedding call to Neosantara formats the request correctly as an OpenAI-like call. + """ + os.environ["NEOSANTARA_API_KEY"] = "sk-1234" + + mock_embedding = Embedding( + embedding=[0.1, 0.2, 0.3], + index=0, + object="embedding" + ) + mock_response = CreateEmbeddingResponse( + data=[mock_embedding], + model="nusa-embedding-0001", + object="list", + usage={"prompt_tokens": 10, "total_tokens": 10} + ) + + mock_create.return_value = MockAPIResponse(mock_response) + + response = litellm.embedding( + model="neosantara/nusa-embedding-0001", + input=["hi"], + api_key="sk-1234" + ) + + assert response.data[0]["embedding"] == [0.1, 0.2, 0.3] + assert response._hidden_params["custom_llm_provider"] == "neosantara" + assert response._hidden_params["api_base"] == "https://api.neosantara.xyz/v1" +@patch("openai.resources.chat.completions.Completions.create") +def test_neosantara_responses_api_bridge(mock_create): + """ + Test that Neosantara works with litellm.responses() API bridge. + """ + os.environ["NEOSANTARA_API_KEY"] = "sk-1234" + + # Define the mock response object to be returned by the patched method + mock_choice = Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + content="Hello from bridge", + role="assistant", + function_call=None, + tool_calls=None, + ) + ) + mock_usage = CompletionUsage( + completion_tokens=10, + prompt_tokens=10, + total_tokens=20 + ) + mock_chat_completion = ChatCompletion( + id="chatcmpl-123", + choices=[mock_choice], + created=1677610602, + model="claude-3-haiku", + object="chat.completion", + usage=mock_usage, + system_fingerprint=None + ) + + mock_create.return_value = MockAPIResponse(mock_chat_completion) + + response = litellm.responses( + model="neosantara/claude-3-haiku", + input="hi from responses", + api_key="sk-1234" + ) + + # Verify result is transformed to Responses API schema + # Bridge transforms choices[0].message.content -> output[0].content[0].text + assert response.model == "neosantara/claude-3-haiku" + assert response.output[0].content[0].text == "Hello from bridge" + assert hasattr(response, "created_at") + assert response.created_at == 1677610602 + +@pytest.mark.anyio +@patch("openai.resources.chat.completions.AsyncCompletions.create") +async def test_neosantara_anthropic_messages_bridge(mock_async_create): + """ + Test that Neosantara works with litellm.anthropic_messages() API bridge. + """ + os.environ["NEOSANTARA_API_KEY"] = "sk-1234" + + # Define the mock response object to be returned by the patched method + mock_choice = Choice( + finish_reason="stop", + index=0, + message=ChatCompletionMessage( + content="Hello from anthropic bridge", + role="assistant", + function_call=None, + tool_calls=None, + ) + ) + mock_usage = CompletionUsage( + completion_tokens=10, + prompt_tokens=10, + total_tokens=20 + ) + mock_chat_completion = ChatCompletion( + id="chatcmpl-123", + choices=[mock_choice], + created=1677610602, + model="claude-3-haiku", + object="chat.completion", + usage=mock_usage, + system_fingerprint=None + ) + + # For async, the mock's return value should be an awaitable + async def side_effect(*args, **kwargs): + return MockAPIResponse(mock_chat_completion) + + mock_async_create.side_effect = side_effect + + response = await litellm.anthropic_messages( + model="neosantara/claude-3-haiku", + messages=[{"role": "user", "content": "hi from anthropic"}], + api_key="sk-1234", + max_tokens=100 + ) + + # Verify result is transformed to Anthropic Messages API schema + assert response["model"] == "neosantara/claude-3-haiku" + assert response["role"] == "assistant" + assert response["content"][0].text == "Hello from anthropic bridge" + assert response["type"] == "message" + assert response["usage"]["input_tokens"] == 10 + assert response["usage"]["output_tokens"] == 10 \ No newline at end of file