diff --git a/litellm/constants.py b/litellm/constants.py index bbeb4846e278..928aaf49a1ea 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -929,6 +929,7 @@ "https://api.cognition.ai/v1", "https://api.scx.ai/v1", "https://gigachat.devices.sberbank.ru/api/v1", + "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", ] @@ -1000,6 +1001,7 @@ "meta", # Meta Model API (Muse Spark) - JSON-configured provider "cognition", "scx-ai", + "streamlake", ] OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers)) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index fe10293c420f..ec9568514088 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -200,5 +200,14 @@ "temperature_max": 1.99 }, "supported_endpoints": ["/v1/chat/completions"] + }, + "streamlake": { + "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "api_key_env": "STREAMLAKE_API_KEY", + "api_base_env": "STREAMLAKE_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "supported_endpoints": ["/v1/chat/completions"] } } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 11da46f8eb32..f9befc4192d3 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -59610,6 +59610,200 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "rpm": 60, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "tpm": 600000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "rpm": 30, + "tpm": 600000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "rpm": 10, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "tpm": 300000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "rpm": 30, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "tpm": 300000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Air-V2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V1": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V2": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V2.5": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 7.4e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 2.96e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-235B-A22B-Instruct-2507": { + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 3.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 126000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.4e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-30B-A3B": { + "cache_read_input_token_cost": 4.3e-08, + "input_cost_per_token": 1.07e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 30000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.071e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-30B-A3B-Instruct-2507": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 1.07e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 126000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4.29e-07, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 9e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-Coder-Next": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 30c1e0b894e1..248b0ba99e9b 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2606,6 +2606,23 @@ "messages": true, "responses": true } + }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } } }, "endpoints": { diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index d7e100dd6300..253b7a989f1c 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2985,6 +2985,34 @@ ], "default_model_placeholder": "scx-ai/GLM-5.2" }, + { + "provider": "StreamLake", + "provider_display_name": "StreamLake", + "litellm_provider": "streamlake", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "streamlake/GLM-5.3-Flash" + }, { "provider": "Snowflake", "provider_display_name": "Snowflake", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc9b7931f566..55684cc8d761 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4175,6 +4175,7 @@ class LlmProviders(str, Enum): SCX_AI = "scx-ai" DARKBLOOM = "darkbloom" META = "meta" + STREAMLAKE = "streamlake" LITELLM_AGENT = "litellm_agent" CURSOR = "cursor" BEDROCK_MANTLE = "bedrock_mantle" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 11da46f8eb32..f9befc4192d3 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -59610,6 +59610,200 @@ "supports_tool_choice": true, "supports_vision": true }, + "streamlake/deepseek-v4-flash-0731": { + "cache_read_input_token_cost": 1.4e-08, + "input_cost_per_token": 4.4e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 1.32e-06, + "rpm": 60, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "tpm": 600000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "cache_read_input_token_cost": 4.4e-08, + "input_cost_per_token": 1.32e-06, + "litellm_provider": "streamlake", + "max_output_tokens": 384000, + "max_tokens": 384000, + "mode": "chat", + "output_cost_per_token": 3.96e-06, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "rpm": 30, + "tpm": 600000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/DeepSeek-V3": { + "cache_read_input_token_cost": 1.14e-07, + "input_cost_per_token": 2.86e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 56000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.143e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3": { + "cache_read_input_token_cost": 2.6e-07, + "input_cost_per_token": 1.4e-06, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 4.4e-06, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "rpm": 10, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "tpm": 300000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/GLM-5.3-Flash": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 1048576, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 5e-07, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_vision": true, + "rpm": 30, + "supports_assistant_prefill": true, + "supports_prompt_caching": true, + "tpm": 300000, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Air-V2.5": { + "cache_read_input_token_cost": 3e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V1": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 256000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V2": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/KAT-Coder-Pro-V2.5": { + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 7.4e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 80000, + "max_tokens": 80000, + "mode": "chat", + "output_cost_per_token": 2.96e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Kimi-K2-Instruct-0905": { + "cache_read_input_token_cost": 1.43e-07, + "input_cost_per_token": 5.71e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 224000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 2.286e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-235B-A22B-Instruct-2507": { + "cache_read_input_token_cost": 7e-08, + "input_cost_per_token": 3.5e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 126000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 1.4e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-30B-A3B": { + "cache_read_input_token_cost": 4.3e-08, + "input_cost_per_token": 1.07e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 30000, + "max_output_tokens": 8000, + "max_tokens": 8000, + "mode": "chat", + "output_cost_per_token": 1.071e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-30B-A3B-Instruct-2507": { + "cache_read_input_token_cost": 2.2e-08, + "input_cost_per_token": 1.07e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 126000, + "max_output_tokens": 32000, + "max_tokens": 32000, + "mode": "chat", + "output_cost_per_token": 4.29e-07, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-Coder-480B-A35B-Instruct-FP8": { + "cache_read_input_token_cost": 1.8e-07, + "input_cost_per_token": 9e-07, + "litellm_provider": "streamlake", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 4.5e-06, + "supports_function_calling": true, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, + "streamlake/Qwen3-Coder-Next": { + "cache_read_input_token_cost": 6e-08, + "input_cost_per_token": 3e-07, + "litellm_provider": "streamlake", + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.5e-06, + "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce" + }, "deepseek-v4-flash": { "cache_creation_input_token_cost": 0.0, "cache_read_input_token_cost": 6e-09, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index af9b194bbeeb..ca1cf8b51409 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2384,6 +2384,23 @@ "text_completion": true } }, + "streamlake": { + "display_name": "StreamLake (`streamlake`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "tencent": { "display_name": "Tencent TokenHub (`tencent`)", "url": "https://docs.litellm.ai/docs/providers/tencent", diff --git a/tests/test_litellm/llms/openai_like/test_streamlake_provider.py b/tests/test_litellm/llms/openai_like/test_streamlake_provider.py new file mode 100644 index 000000000000..719dc1594a5f --- /dev/null +++ b/tests/test_litellm/llms/openai_like/test_streamlake_provider.py @@ -0,0 +1,506 @@ +import json +from pathlib import Path +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + +STREAMLAKE_BASE_URL = "https://vanchin.streamlake.ai/api/gateway/v1/endpoints" +STREAMLAKE_CHAT_URL = f"{STREAMLAKE_BASE_URL}/chat/completions" + +STREAMLAKE_MODELS = ( + ("streamlake/KAT-Coder-Pro-V2.5", 7.4e-07, 2.96e-06, 1.5e-07, None, 80000, 80000, None, None), + ("streamlake/GLM-5.3", 1.4e-06, 4.4e-06, 2.6e-07, 1048576, 128000, 128000, 10, 300000), + ("streamlake/GLM-5.3-Flash", 1.5e-07, 5e-07, 3e-08, 1048576, 128000, 128000, 30, 300000), + ("streamlake/KAT-Coder-Air-V2.5", 1.5e-07, 6e-07, 3e-08, None, 80000, 80000, None, None), + ("streamlake/Qwen3-Coder-Next", 3e-07, 1.5e-06, 6e-08, None, 64000, 64000, None, None), + ("streamlake/KAT-Coder-Pro-V2", 3e-07, 1.2e-06, 6e-08, None, 80000, 80000, None, None), + ("streamlake/Qwen3-Coder-480B-A35B-Instruct-FP8", 9e-07, 4.5e-06, 1.8e-07, 200000, 64000, 64000, None, None), + ("streamlake/Qwen3-235B-A22B-Instruct-2507", 3.5e-07, 1.4e-06, 7e-08, 126000, 32000, 32000, None, None), + ("streamlake/Qwen3-30B-A3B-Instruct-2507", 1.07e-07, 4.29e-07, 2.2e-08, 126000, 32000, 32000, None, None), + ("streamlake/Kimi-K2-Instruct-0905", 5.71e-07, 2.286e-06, 1.43e-07, 224000, 32000, 32000, None, None), + ("streamlake/DeepSeek-V3", 2.86e-07, 1.143e-06, 1.14e-07, 56000, 8000, 8000, None, None), + ("streamlake/DeepSeek-V4-Pro-0813", 1.32e-06, 3.96e-06, 4.4e-08, None, 384000, 384000, 30, 600000), + ("streamlake/deepseek-v4-flash-0731", 4.4e-07, 1.32e-06, 1.4e-08, None, 384000, 384000, 60, 600000), + ("streamlake/Qwen3-30B-A3B", 1.07e-07, 1.071e-06, 4.3e-08, 30000, 8000, 8000, None, None), + ("streamlake/KAT-Coder-Pro-V1", 3e-07, 1.2e-06, 6e-08, 256000, 32000, 32000, None, None), +) + +PUBLIC_MODEL_IDS = tuple(row[0].split("/", 1)[1] for row in STREAMLAKE_MODELS) + +STREAMLAKE_CAPABILITIES = { + "streamlake/GLM-5.3": {"reasoning", "function_calling", "response_schema", "prefill", "prompt_caching", "text"}, + "streamlake/GLM-5.3-Flash": { + "reasoning", + "function_calling", + "response_schema", + "prefill", + "prompt_caching", + "vision", + "text", + }, + "streamlake/DeepSeek-V4-Pro-0813": { + "reasoning", + "function_calling", + "response_schema", + "prompt_caching", + "text", + }, + "streamlake/deepseek-v4-flash-0731": {"reasoning", "function_calling", "response_schema", "text"}, +} + + +class TestStreamlakeProviderConfig: + def test_streamlake_in_provider_list(self): + from litellm import LlmProviders + + assert hasattr(LlmProviders, "STREAMLAKE") + assert LlmProviders.STREAMLAKE.value == "streamlake" + assert "streamlake" in litellm.provider_list + + def test_streamlake_json_config_exists(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + assert JSONProviderRegistry.exists("streamlake") + + streamlake = JSONProviderRegistry.get("streamlake") + assert streamlake is not None + assert streamlake.base_url == STREAMLAKE_BASE_URL + assert streamlake.api_key_env == "STREAMLAKE_API_KEY" + assert streamlake.api_base_env == "STREAMLAKE_API_BASE" + + def test_streamlake_is_chat_completions_only(self): + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + streamlake = JSONProviderRegistry.get("streamlake") + assert streamlake is not None + assert streamlake.supported_endpoints == ["/v1/chat/completions"] + assert JSONProviderRegistry.supports_responses_api("streamlake") is False + + def test_streamlake_in_openai_compatible_providers(self): + from litellm.constants import openai_compatible_providers + + assert "streamlake" in openai_compatible_providers + + def test_streamlake_provider_resolution(self): + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + assert api_key == "sk-test" + + def test_streamlake_api_base_override(self): + model, provider, api_key, api_base = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base="https://custom.streamlake.example/v1", + api_key="sk-test", + ) + + assert model == "ep-test-endpoint" + assert provider == "streamlake" + assert api_base == "https://custom.streamlake.example/v1" + assert api_key == "sk-test" + + def test_streamlake_url_autodetection(self): + model, provider, api_key, api_base = get_llm_provider( + model="some-public-model", + custom_llm_provider=None, + api_base=STREAMLAKE_BASE_URL, + api_key=None, + ) + assert provider == "streamlake" + assert api_base == STREAMLAKE_BASE_URL + + def test_streamlake_api_key_resolved_from_env(self, monkeypatch): + monkeypatch.setenv("STREAMLAKE_API_KEY", "sk-env-key") + + _, provider, api_key, _ = get_llm_provider( + model="streamlake/ep-test-endpoint", + custom_llm_provider=None, + api_base=None, + api_key=None, + ) + assert provider == "streamlake" + assert api_key == "sk-env-key" + + def test_streamlake_validate_environment_sets_bearer(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("streamlake") + assert provider is not None + config = create_config_class(provider)() + + headers = config.validate_environment( + headers={}, + model="ep-test-endpoint", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + litellm_params={}, + api_key="sk-resolved-key", + api_base=None, + ) + assert headers["Authorization"] == "Bearer sk-resolved-key" + + def test_streamlake_complete_url_appends_chat_completions(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("streamlake") + assert provider is not None + config = create_config_class(provider)() + + url = config.get_complete_url( + api_base=None, + api_key="sk-test", + model="ep-test-endpoint", + optional_params={}, + litellm_params={}, + ) + assert url == STREAMLAKE_CHAT_URL + + def test_streamlake_max_completion_tokens_mapped_to_max_tokens(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("streamlake") + assert provider is not None + config = create_config_class(provider)() + + optional_params = config.map_openai_params( + non_default_params={"max_completion_tokens": 1024}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"max_tokens": 1024} + assert "max_completion_tokens" not in optional_params + + def test_streamlake_chat_params_pass_through(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("streamlake") + assert provider is not None + config = create_config_class(provider)() + + optional_params = config.map_openai_params( + non_default_params={"temperature": 0.2, "top_p": 0.9, "stream": True}, + optional_params={}, + model="ep-test-endpoint", + drop_params=False, + ) + assert optional_params == {"temperature": 0.2, "top_p": 0.9, "stream": True} + + def test_streamlake_does_not_advertise_tools_or_reasoning_params(self): + from litellm.llms.openai_like.dynamic_config import create_config_class + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider = JSONProviderRegistry.get("streamlake") + assert provider is not None + config = create_config_class(provider)() + + for model in ("ep-test-endpoint", "KAT-Coder-Pro-V2"): + supported = config.get_supported_openai_params(model=model) + assert "tools" not in supported + assert "tool_choice" not in supported + assert "reasoning_effort" not in supported + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "is_async, api_base", + ( + (False, None), + (False, "https://custom.streamlake.example/v1"), + (False, "https://custom.streamlake.example/v1/"), + (True, None), + (True, "https://custom.streamlake.example/v1"), + (True, "https://custom.streamlake.example/v1/"), + ), + ids=( + "sync-default", + "sync-override", + "sync-override-trailing-slash", + "async-default", + "async-override", + "async-override-trailing-slash", + ), +) +async def test_streamlake_completion_uses_openai_compatible_http( + is_async: bool, api_base: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + response_body: Final[dict[str, object]] = { + "id": "chatcmpl-streamlake-test", + "object": "chat.completion", + "created": 1710000000, + "model": "ep-test-endpoint", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "StreamLake response"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + } + expected_url: Final[str] = f"{(api_base or STREAMLAKE_BASE_URL).rstrip('/')}/chat/completions" + expected_request_body: Final[dict[str, object]] = { + "messages": [{"role": "user", "content": "hello"}], + "model": "ep-test-endpoint", + "max_tokens": 512, + } + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert str(request.url) == expected_url + assert request.headers["authorization"] == "Bearer sk-streamlake-test" + assert request.headers["content-type"] == "application/json" + assert json.loads(request.content) == expected_request_body + return httpx.Response(200, json=response_body, request=request) + + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=True) as respx_mock: + route: Final = respx_mock.post(expected_url).mock(side_effect=handler) + if is_async: + response: Final = await litellm.acompletion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_completion_tokens=512, + ) + else: + response: Final = litellm.completion( + model="streamlake/ep-test-endpoint", + messages=[{"role": "user", "content": "hello"}], + api_key="sk-streamlake-test", + api_base=api_base, + max_completion_tokens=512, + ) + + assert route.call_count == 1 + assert len(response.choices) == 1 + assert response.choices[0].message.role == "assistant" + assert response.choices[0].message.content == "StreamLake response" + assert response.usage is not None + assert response.usage.prompt_tokens == 3 + assert response.usage.completion_tokens == 2 + assert response.usage.total_tokens == 5 + + +class TestStreamlakeNoHardcodedEndpointIds: + def test_provider_config_never_hardcodes_an_endpoint_id(self): + path = Path(litellm.__file__).parent / "llms" / "openai_like" / "providers.json" + with open(path) as f: + providers = json.load(f) + + streamlake_config = providers.get("streamlake") + assert streamlake_config is not None + serialized = json.dumps(streamlake_config) + assert "ep-" not in serialized, "provider config must not embed any Inference Endpoint ID" + + def test_registered_public_models_are_only_the_aligned_scope(self): + path = Path(litellm.__file__).parent / "model_prices_and_context_window_backup.json" + with open(path) as f: + model_cost = json.load(f) + + streamlake_keys = {key for key in model_cost if key.startswith("streamlake/")} + assert streamlake_keys == {f"streamlake/{model}" for model in PUBLIC_MODEL_IDS} + + def test_registry_keys_contain_no_whitespace(self): + path = Path(litellm.__file__).parent / "model_prices_and_context_window_backup.json" + with open(path) as f: + model_cost = json.load(f) + + streamlake_keys = [key for key in model_cost if key.startswith("streamlake/")] + assert streamlake_keys + for key in streamlake_keys: + assert " " not in key, f"model registry keys must not contain spaces: {key}" + + +class TestStreamlakeModelMetadata: + @staticmethod + def _load(*path_parts): + path = Path(__file__).parents[4].joinpath(*path_parts) + with open(path) as f: + return json.load(f) + + @pytest.mark.parametrize( + "model, input_cost, output_cost, cache_cost, max_input, max_output, max_tokens, rpm, tpm", + STREAMLAKE_MODELS, + ids=[model.split("/", 1)[1] for model, *_ in STREAMLAKE_MODELS], + ) + def test_model_entry_pricing_and_limits( + self, model, input_cost, output_cost, cache_cost, max_input, max_output, max_tokens, rpm, tpm + ): + entry = self._load("model_prices_and_context_window.json")[model] + + assert entry["litellm_provider"] == "streamlake" + assert entry["mode"] == "chat" + + assert entry["input_cost_per_token"] == input_cost + assert entry["output_cost_per_token"] == output_cost + assert entry["cache_read_input_token_cost"] == cache_cost + + if max_input is None: + assert "max_input_tokens" not in entry + else: + assert entry["max_input_tokens"] == max_input + assert entry["max_output_tokens"] == max_output + assert entry["max_tokens"] == max_tokens + if model in STREAMLAKE_CAPABILITIES: + assert entry["rpm"] == rpm + assert entry["tpm"] == tpm + else: + assert rpm is None + assert tpm is None + + @pytest.mark.parametrize("model, capabilities", STREAMLAKE_CAPABILITIES.items()) + def test_new_model_capabilities(self, model, capabilities): + entry = self._load("model_prices_and_context_window.json")[model] + assert entry.get("supports_reasoning") is ("reasoning" in capabilities) + assert entry.get("supports_function_calling") is ("function_calling" in capabilities) + assert entry.get("supports_response_schema") is ("response_schema" in capabilities) + assert entry.get("supports_vision", False) is ("vision" in capabilities) + assert entry.get("supports_prompt_caching", False) is ("prompt_caching" in capabilities) + assert entry.get("supports_assistant_prefill", False) is ("prefill" in capabilities) + + @pytest.mark.parametrize( + "model, max_output_tokens", + ( + ("streamlake/GLM-5.3", 128000), + ("streamlake/GLM-5.3-Flash", 128000), + ("streamlake/DeepSeek-V4-Pro-0813", 384000), + ("streamlake/deepseek-v4-flash-0731", 384000), + ), + ) + def test_get_model_info_resolves_new_models(self, local_model_cost_map, model, max_output_tokens): + info = litellm.get_model_info(model=model) + assert info["litellm_provider"] == "streamlake" + assert info["mode"] == "chat" + assert info["max_output_tokens"] == max_output_tokens + assert info["max_tokens"] == max_output_tokens + assert ( + info["rpm"] + == { + "streamlake/GLM-5.3": 10, + "streamlake/GLM-5.3-Flash": 30, + "streamlake/DeepSeek-V4-Pro-0813": 30, + "streamlake/deepseek-v4-flash-0731": 60, + }[model] + ) + assert ( + info["tpm"] + == { + "streamlake/GLM-5.3": 300000, + "streamlake/GLM-5.3-Flash": 300000, + "streamlake/DeepSeek-V4-Pro-0813": 600000, + "streamlake/deepseek-v4-flash-0731": 600000, + }[model] + ) + assert info["supports_function_calling"] is True + assert info["supports_reasoning"] is True + assert info["supports_response_schema"] is True + assert ( + info.get("supports_prompt_caching") is None + if model == "streamlake/deepseek-v4-flash-0731" + else info["supports_prompt_caching"] is True + ) + assert ( + info.get("supports_assistant_prefill", False) is True + if model.startswith("streamlake/GLM") + else info.get("supports_assistant_prefill") is None + ) + + model_cost = self._load("model_prices_and_context_window.json") + backup = self._load("litellm", "model_prices_and_context_window_backup.json") + for model, *_ in STREAMLAKE_MODELS: + assert model in backup, f"{model} missing from backup json" + assert backup[model] == model_cost[model], f"{model} differs between root and backup json" + + def test_minimal_scope_metadata_only(self): + model_cost = self._load("model_prices_and_context_window.json") + function_calling_models = { + model + for model, *_ in STREAMLAKE_MODELS + if model not in {"streamlake/Qwen3-Coder-Next", "streamlake/KAT-Coder-Pro-V2"} + } + for model, *_ in STREAMLAKE_MODELS: + entry = model_cost[model] + if model in function_calling_models: + assert entry.get("supports_function_calling") is True + else: + assert "supports_function_calling" not in entry + for capability in ( + "supports_parallel_function_calling", + "supports_tool_choice", + ): + assert capability not in entry, f"{model} must not advertise {capability}" + if model not in STREAMLAKE_CAPABILITIES: + for capability in ( + "supports_vision", + "supports_reasoning", + "supports_response_schema", + "supports_prompt_caching", + ): + assert capability not in entry, f"{model} must not advertise {capability}" + + def test_get_model_info_resolves_public_model_metadata(self, local_model_cost_map): + info = litellm.get_model_info(model="streamlake/KAT-Coder-Pro-V2.5") + assert info["litellm_provider"] == "streamlake" + assert info["input_cost_per_token"] == 7.4e-07 + assert info["output_cost_per_token"] == 2.96e-06 + assert info["cache_read_input_token_cost"] == 1.5e-07 + assert info["max_output_tokens"] == 80000 + assert info["max_tokens"] == 80000 + assert info["mode"] == "chat" + + def test_completion_cost_uses_streamlake_pricing(self, local_model_cost_map): + from litellm import completion_cost + from litellm.types.utils import ModelResponse, Usage + + response = ModelResponse( + model="KAT-Coder-Pro-V2.5", + usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), + ) + cost = completion_cost( + completion_response=response, + model="streamlake/KAT-Coder-Pro-V2.5", + custom_llm_provider="streamlake", + ) + expected = 1000 * 7.4e-07 + 500 * 2.96e-06 + assert abs(cost - expected) < 1e-12 + + def test_completion_cost_uses_lowercase_provider_response_model(self, local_model_cost_map): + from litellm import completion_cost + from litellm.types.utils import ModelResponse, Usage + + response = ModelResponse( + model="deepseek-v4-flash-0731", + usage=Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500), + ) + cost = completion_cost( + completion_response=response, + model="streamlake/ep-test-endpoint", + custom_llm_provider="streamlake", + base_model="streamlake/deepseek-v4-flash-0731", + ) + expected = 1000 * 4.4e-07 + 500 * 1.32e-06 + assert abs(cost - expected) < 1e-12 + + def test_qwen3_30b_a3b_uses_highest_published_output_price(self): + entry = self._load("model_prices_and_context_window.json")["streamlake/Qwen3-30B-A3B"] + assert entry["output_cost_per_token"] == 1.071e-06 diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 680dd4df0aea..083ad56e8f93 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -963,6 +963,29 @@ def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache) assert len(chat["providers"]) > 0 +def test_get_supported_endpoints_streamlake_only_advertises_chat_completions( + reset_endpoints_cache: None, +) -> None: + app: Final[FastAPI] = FastAPI() + app.include_router(router) + client: Final[TestClient] = TestClient(app) + with client: + response: Final = client.get("/public/endpoints") + assert response.status_code == 200, response.text + + endpoints: Final = response.json()["endpoints"] + streamlake_endpoints: Final = { + endpoint["key"] + for endpoint in endpoints + if any(provider["slug"] == "streamlake" for provider in endpoint["providers"]) + } + assert streamlake_endpoints == {"chat_completions"} + + chat_completions: Final = next(endpoint for endpoint in endpoints if endpoint["key"] == "chat_completions") + assert chat_completions["endpoint"] == "/chat/completions" + assert "streamlake" in {provider["slug"] for provider in chat_completions["providers"]} + + def test_get_supported_endpoints_display_names_have_no_slug_suffix( reset_endpoints_cache, ): diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index efba26734ff4..bf6e04c87cb3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -22,7 +22,13 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => usePtuCostAttributionEnabled: () => mockPtuEnabled(), })); -vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: () => ({ data: {} }) })); +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: () => ({ + data: { + "streamlake/GLM-5.3": { litellm_provider: "streamlake" }, + }, + }), +})); vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({ useCredentials: () => ({ data: { credentials: [] } }), @@ -60,6 +66,13 @@ vi.mock("@/app/(dashboard)/hooks/providers/useProviderFields", () => ({ { key: "api_base", label: "API Base", field_type: "text", required: false }, ], }, + { + provider: "StreamLake", + provider_display_name: "StreamLake", + litellm_provider: "streamlake", + default_model_placeholder: "streamlake/", + credential_fields: [{ key: "api_key", label: "API Key", field_type: "password", required: false }], + }, ], isLoading: false, error: null, @@ -154,6 +167,27 @@ describe("AddModelPanel submit payload contract", () => { }); }); + it("creates a StreamLake model from a public model selection and API key", async () => { + const { user, submit } = await setup(); + + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("StreamLake")); + await user.click(screen.getByRole("combobox", { name: "Select models" })); + await user.click(await screen.findByText("streamlake/GLM-5.3")); + await user.type(screen.getByLabelText("API Key"), "sk-streamlake-test"); + await submit(); + + expect(lastCreatedModel()).toStrictEqual({ + model_name: "streamlake/GLM-5.3", + litellm_params: { + model: "streamlake/GLM-5.3", + custom_llm_provider: "streamlake", + api_key: "sk-streamlake-test", + }, + model_info: { ...baseModelInfo }, + }); + }); + it("registers four more keys as undefined once Advanced Settings opens", async () => { const { openAdvanced, fillRequired, submit } = await setup(); await fillRequired(); diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 4c68e3022679..207c6744c665 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -176,6 +176,7 @@ describe("provider_info_helpers", () => { Providers.PETALS, Providers.PG_VECTOR, Providers.PREDIBASE, + Providers.StreamLake, Providers.WANDB, Providers.ZAI, ]; @@ -328,6 +329,19 @@ describe("provider_info_helpers", () => { expect(result).toEqual(["gpt-3.5-turbo", "gpt-4"]); }); + it("should return StreamLake models and exclude models from other providers", () => { + const modelMap = { + "streamlake/GLM-5.3": { litellm_provider: "streamlake" }, + "streamlake/DeepSeek-V3": { litellm_provider: "streamlake" }, + "gpt-4o": { litellm_provider: "openai" }, + }; + const result = getProviderModels(Providers.StreamLake, modelMap); + + expect(provider_map.StreamLake).toBe("streamlake"); + expect(result).toEqual(["streamlake/GLM-5.3", "streamlake/DeepSeek-V3"]); + expect(result).not.toContain("gpt-4o"); + }); + it("should return models whose litellm_provider is a prefix-anchored variant of the provider", () => { const modelMap = { "anthropic-text-model": { litellm_provider: "anthropic_text" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 1a1a2fd73aa0..3e7247f4bda1 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -163,6 +163,7 @@ export enum Providers { SCX_AI = "SCX.ai", Snowflake = "Snowflake", Soniox = "Soniox", + StreamLake = "StreamLake", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", TogetherAI = "TogetherAI", TOPAZ = "Topaz", @@ -280,6 +281,7 @@ export const provider_map: Record = { SCX_AI: "scx-ai", Snowflake: "snowflake", Soniox: "soniox", + StreamLake: "streamlake", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", TogetherAI: "together_ai", TOPAZ: "topaz", @@ -445,6 +447,7 @@ const providerPlaceholderMap: Partial> = { [Providers.SageMaker]: "sagemaker/jumpstart-dft-meta-textgeneration-llama-2-7b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", + [Providers.StreamLake]: "streamlake/", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/",