From 97894f1603942ab4784855ba9fd741722a65ac5b Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Thu, 14 Aug 2025 10:22:17 +0800 Subject: [PATCH 1/7] add ModelScope API support --- docs/my-website/docs/providers/modelscope.md | 242 ++++++++++++++++++ litellm/__init__.py | 39 ++- litellm/constants.py | 88 ++++--- .../get_llm_provider_logic.py | 10 + .../llms/modelscope/chat/transformation.py | 75 ++++++ litellm/types/utils.py | 1 + litellm/utils.py | 6 + .../test_modelscope_chat_transformation.py | 114 +++++++++ 8 files changed, 531 insertions(+), 44 deletions(-) create mode 100644 docs/my-website/docs/providers/modelscope.md create mode 100644 litellm/llms/modelscope/chat/transformation.py create mode 100644 tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py diff --git a/docs/my-website/docs/providers/modelscope.md b/docs/my-website/docs/providers/modelscope.md new file mode 100644 index 000000000000..b88041ae176e --- /dev/null +++ b/docs/my-website/docs/providers/modelscope.md @@ -0,0 +1,242 @@ +# ModelScope +LiteLLM supports running inference across multiple services for models hosted on the ModelScope Hub. + +## Supported Models + +### Serverless Inference Providers +You can check available models for an inference provider by going to [modelscope.cn/models](https://modelscope.cn/models), clicking the "API-Inference" and the "Other" filter tab, and selecting your desired provider. + +For example, you can find all Qwen3 series models [here](https://modelscope.cn/models?filter=inference_type&model_type=qwen3&page=1&tabKey=other). + + +### Dedicated Inference Endpoints +Refer to the [Inference Endpoints catalog](https://modelscope.cn/models?filter=inference_type&page=1&tabKey=task) for a list of available models. + +## Usage + + +### Authentication +With a single ModelScope token, you can access inference through multiple providers. Your calls are routed through ModelScope and the usage is free. +However, please ensure you bind your Alibaba Cloud account before use. For details, refer to the following two links. +- [API-Infereference](https://modelscope.cn/docs/model-service/API-Inference/intro) +- [Binding Alibaba Cloud Account](https://modelscope.cn/docs/accounts/aliyun-binding-and-authorization) + +Simply set the `MODELSCOPE_TOKEN` environment variable with your ModelScope token, you can create one here: https://modelscope.cn/my/myaccesstoken. + +```bash +export MODELSCOPE_TOKEN="123xxxxxx" +``` +or alternatively, you can pass your ModelScope token as a parameter: +```python +completion(..., api_key="123xxxxxx") +``` + +### Getting Started + +To use a ModelScope model, specify the model you want to use in the following format: +``` +// +``` +Where `/` is the ModelScope model ID. + +Examples: + +```python +# Run Llama-4-Scout-17B-16E-Instruct inference through LLM-Research +completion(model="modelscope/LLM-Research/Llama-4-Scout-17B-16E-Instruct",...) + +# Run DeepSeek-R1 inference through DeepSeek +completion(model="modelscope/deepseek-ai/DeepSeek-R1-0528",...) + +# Run Qwen3-8B inference through Qwen +completion(model="modelscope/Qwen/Qwen3-8B",...) +``` + + +### Basic Completion +Here's an example of chat completion using the Qwen3-8B model through Qwen: + +```python +import os +from litellm import completion + +os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" + +response = completion( + model="modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct", + messages=[ + { + "role": "user", + "content": "How many r's are in the word 'strawberry'?", + } + ], +) +print(response) +``` + +### Streaming +Now, let's see what a streaming request looks like. + +```python +import os +from litellm import completion + +os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" + +response = completion( + model="modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct", + messages=[ + { + "role": "user", + "content": "How many r's are in the word `strawberry`?", + + } + ], + stream=True, +) + +for chunk in response: + print(chunk) +``` + +### Image Input +You can also pass images when the model supports it. Here is an example using [Qwen/Qwen2.5-VL-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-72B-Instruct) model. + +```python +from litellm import completion + +# Set your ModelScope Token +os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" + +messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/audrey_hepburn.jpg" + } + } + ] + } + ] + +response = completion( + model="modelscope/Qwen/Qwen2.5-VL-72B-Instruct", + messages=messages, +) +print(response.choices[0]) +``` + +## Model Deployment +SwingDeploy deployment service is a one-stop model deployment solution launched by ModelScope, aiming to provide developers with end-to-end services from model selection to cloud deployment. Through standardized deployment processes and cloud resource adaptation capabilities, users can quickly deploy the rich models of the Moda community (in multiple fields such as voice, video, and NLP) to the target cloud environment, achieving efficient implementation of model inference services. You can refer to [SwingDeploy](https://modelscope.cn/docs/model-service/deployment/intro) for more details. + +## LiteLLM Proxy Server with ModelScope models +You can set up a [LiteLLM Proxy Server](https://docs.litellm.ai/#litellm-proxy-server-llm-gateway) to serve ModelScope models through any of the supported Inference Providers. Here's how to do it: + +### Step 1. Setup the config file + +In this case, we are configuring a proxy to serve `Qwen3-Coder-480B-A35B-Instruct` from ModelScope. + +```yaml +model_list: + - model_name: my-model + litellm_params: + model: modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct + api_key: os.environ/MODELSCOPE_TOKEN # ensure you have `MODELSCOPE_TOKEN` in your .env +``` + +### Step 2. Start the server +```bash +litellm --config /path/to/config.yaml +``` + +### Step 3. Make a request to the server + + + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "my-model", + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ] +}' +``` + + +## Usage with LiteLLM Proxy Server + +Here's how to call a ModelScope model with the LiteLLM Proxy Server + +1. Modify the config.yaml + + ```yaml showLineNumbers + model_list: + - model_name: my-model + litellm_params: + model: modelscope// # add modelscope/ prefix to route as ModelScope provider + api_key: api-key # api key to send your model + ``` + + +2. Start the proxy + + ```bash + $ litellm --config /path/to/config.yaml + ``` + +3. Send Request to LiteLLM Proxy Server + + + + + + ```python showLineNumbers + import openai + client = openai.OpenAI( + api_key="123xxxx", # pass litellm proxy key, if you're using virtual keys + base_url="http://0.0.0.0:4000" # litellm-proxy-base url + ) + + response = client.chat.completions.create( + model="my-model", + messages = [ + { + "role": "user", + "content": "what llm are you" + } + ], + ) + + print(response) + ``` + + + + + ```shell + curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer 1234xxxxx' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "my-model", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + }' + ``` + + + + diff --git a/litellm/__init__.py b/litellm/__init__.py index c954f5fd31ef..405a83eea923 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -625,6 +625,7 @@ def identify(event_details): deepgram_models: Set = set() elevenlabs_models: Set = set() dashscope_models: Set = set() +modelscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() v0_models: Set = set() @@ -872,7 +873,9 @@ def add_known_models(model_cost_map: Optional[Dict] = None): elif value.get("litellm_provider") == "heroku": heroku_models.add(key) elif value.get("litellm_provider") == "dashscope": - dashscope_models.add(key) + dashscope_models.append(key) + elif value.get("litellm_provider") == "modelscope": + modelscope_models.append(key) elif value.get("litellm_provider") == "moonshot": moonshot_models.add(key) elif value.get("litellm_provider") == "publicai": @@ -992,6 +995,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None): | zai_models | fal_ai_models | deepseek_models + | modelscope_models | azure_ai_models | voyage_models | infinity_models @@ -1123,6 +1127,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None): "elevenlabs": elevenlabs_models, "heroku": heroku_models, "dashscope": dashscope_models, + "modelscope": modelscope_models, "moonshot": moonshot_models, "publicai": publicai_models, "v0": v0_models, @@ -1234,14 +1239,30 @@ def add_known_models(model_cost_map: Optional[Dict] = None): # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo -# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) -# All remaining configs are now lazy loaded - see _lazy_imports_registry.py - -# Import LlmProviders here (before main import) because it's imported during import time -# in multiple places including openai.py (via main import) -from litellm.types.utils import LlmProviders - -## Lazy loading this is not straightforward, will leave it here for now. +from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig +from .llms.azure.completion.transformation import AzureOpenAITextConfig +from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig +from .llms.llamafile.chat.transformation import LlamafileChatConfig +from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig +from .llms.vllm.completion.transformation import VLLMConfig +from .llms.deepseek.chat.transformation import DeepSeekChatConfig +from .llms.lm_studio.chat.transformation import LMStudioChatConfig +from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig +from .llms.nscale.chat.transformation import NscaleConfig +from .llms.perplexity.chat.transformation import PerplexityChatConfig +from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config +from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig +from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig +from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig +from .llms.github_copilot.chat.transformation import GithubCopilotConfig +from .llms.nebius.chat.transformation import NebiusConfig +from .llms.dashscope.chat.transformation import DashScopeChatConfig +from .llms.modelscope.chat.transformation import ModelScopeChatConfig +from .llms.moonshot.chat.transformation import MoonshotChatConfig +from .llms.v0.chat.transformation import V0ChatConfig +from .llms.morph.chat.transformation import MorphChatConfig +from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig +from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .main import * # type: ignore from .compression import compress # type: ignore[no-redef] diff --git a/litellm/constants.py b/litellm/constants.py index 26e25d0cef38..3a6a6c8a31ba 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -614,6 +614,7 @@ "nscale", "nebius", "dashscope", + "modelscope", "moonshot", "publicai", "v0", @@ -770,6 +771,7 @@ "inference.api.nscale.com/v1", "api.studio.nebius.ai/v1", "https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + "https://api-inference.modelscope.cn/v1", "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", "https://api.synthetic.new/openai/v1", @@ -833,6 +835,7 @@ "nscale", "nebius", "dashscope", + "modelscope", "moonshot", "v0", "helicone", @@ -858,6 +861,7 @@ "featherless_ai", "nebius", "dashscope", + "modelscope", "moonshot", "publicai", "synthetic", @@ -1081,42 +1085,56 @@ ] ) -nebius_embedding_models: set = set( - [ - "BAAI/bge-en-icl", - "BAAI/bge-multilingual-gemma2", - "intfloat/e5-mistral-7b-instruct", - ] -) +modelscope_models: List = [ + "LLM-Research/c4ai-command-r-plus-08-2024", + "mistralai/Mistral-Small-Instruct-2409", + "mistralai/Ministral-8B-Instruct-2410", + "mistralai/Mistral-Large-Instruct-2407", + "Qwen/Qwen2.5-Coder-32B-Instruct", + "Qwen/Qwen2.5-Coder-14B-Instruct", + "Qwen/Qwen2.5-Coder-7B-Instruct", + "Qwen/Qwen2.5-72B-Instruct", + "Qwen/Qwen2.5-32B-Instruct", + "Qwen/Qwen2.5-14B-Instruct", + "Qwen/Qwen2.5-7B-Instruct", + "Qwen/QwQ-32B-Preview", + "opencompass/CompassJudger-1-32B-Instruct", + "Qwen/QVQ-72B-Preview", + "Qwen/Qwen2-VL-7B-Instruct", + "Qwen/Qwen2.5-14B-Instruct-1M", + "Qwen/Qwen2.5-7B-Instruct-1M", + "Qwen/Qwen2.5-VL-3B-Instruct", + "Qwen/Qwen2.5-VL-7B-Instruct", + "Qwen/Qwen2.5-VL-72B-Instruct", + "deepseek-ai/DeepSeek-V3", + "Qwen/QwQ-32B", + "XGenerationLab/XiYanSQL-QwenCoder-32B-2412", + "Qwen/Qwen2.5-VL-32B-Instruct", + "LLM-Research/Llama-4-Scout-17B-16E-Instruct", + "LLM-Research/Llama-4-Maverick-17B-128E-Instruct", + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-1.7B", + "Qwen/Qwen3-4B", + "Qwen/Qwen3-8B", + "Qwen/Qwen3-14B", + "Qwen/Qwen3-30B-A3B", + "Qwen/Qwen3-32B", + "Qwen/Qwen3-235B-A22B", + "deepseek-ai/DeepSeek-R1-0528", + "MiniMax/MiniMax-M1-80k", + "Qwen/Qwen3-235B-A22B-Instruct-2507", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "ZhipuAI/GLM-4.5", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", +] -WANDB_MODELS: set = set( - [ - # openai models - "openai/gpt-oss-120b", - "openai/gpt-oss-20b", - # zai-org models - "zai-org/GLM-4.5", - # Qwen models - "Qwen/Qwen3-235B-A22B-Instruct-2507", - "Qwen/Qwen3-Coder-480B-A35B-Instruct", - "Qwen/Qwen3-235B-A22B-Thinking-2507", - # moonshotai - "moonshotai/Kimi-K2-Instruct", - "moonshotai/Kimi-K2.5", - # MiniMaxAI - "MiniMaxAI/MiniMax-M2.5", - # meta models - "meta-llama/Llama-3.1-8B-Instruct", - "meta-llama/Llama-3.3-70B-Instruct", - "meta-llama/Llama-4-Scout-17B-16E-Instruct", - # deepseek-ai - "deepseek-ai/DeepSeek-V3.1", - "deepseek-ai/DeepSeek-R1-0528", - "deepseek-ai/DeepSeek-V3-0324", - # microsoft - "microsoft/Phi-4-mini-instruct", - ] -) +nebius_embedding_models: List = [ + "BAAI/bge-en-icl", + "BAAI/bge-multilingual-gemma2", + "intfloat/e5-mistral-7b-instruct", +] BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "cohere", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a71000f00f87..7b9e575e030b 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -334,6 +334,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1": custom_llm_provider = "dashscope" dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY") + elif endpoint == "api-inference.modelscope.cn/v1": + custom_llm_provider = "modelscope" + dynamic_api_key = get_secret_str("MODELSCOPE_API_KEY") elif endpoint == "api.moonshot.ai/v1": custom_llm_provider = "moonshot" dynamic_api_key = get_secret_str("MOONSHOT_API_KEY") @@ -921,6 +924,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.DashScopeChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "modelscope": + ( + api_base, + dynamic_api_key, + ) = litellm.ModelScopeChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "moonshot": ( api_base, diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py new file mode 100644 index 000000000000..b657bc8950f3 --- /dev/null +++ b/litellm/llms/modelscope/chat/transformation.py @@ -0,0 +1,75 @@ +""" +Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/completions` +""" + +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload + +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + handle_messages_with_content_list_to_str_conversion, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues + +from ...openai.chat.gpt_transformation import OpenAIGPTConfig + + +class ModelScopeChatConfig(OpenAIGPTConfig): + @overload + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: Literal[True] + ) -> Coroutine[Any, Any, List[AllMessageValues]]: ... + + @overload + def _transform_messages( + self, + messages: List[AllMessageValues], + model: str, + is_async: Literal[False] = False, + ) -> List[AllMessageValues]: ... + + def _transform_messages( + self, messages: List[AllMessageValues], model: str, is_async: bool = False + ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: + """ + ModelScope does not support content in list format. + """ + messages = handle_messages_with_content_list_to_str_conversion(messages) + if is_async: + return super()._transform_messages( + messages=messages, model=model, is_async=True + ) + else: + return super()._transform_messages( + messages=messages, model=model, is_async=False + ) + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + api_base = ( + api_base + or get_secret_str("MODELSCOPE_API_BASE") + or "https://api-inference.modelscope.cn/v1" + ) # type: ignore + dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY") + return api_base, dynamic_api_key + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + If api_base is not provided, use the default ModelScope /chat/completions endpoint. + """ + if not api_base: + api_base = "https://api-inference.modelscope.cn/v1" + + if not api_base.endswith("/chat/completions"): + api_base = f"{api_base}/chat/completions" + + return api_base diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 3dcff2be689d..5dc0ece37fe1 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3296,6 +3296,7 @@ class LlmProviders(str, Enum): CODESTRAL = "codestral" TEXT_COMPLETION_CODESTRAL = "text-completion-codestral" DASHSCOPE = "dashscope" + MODELSCOPE = "modelscope" MOONSHOT = "moonshot" PUBLICAI = "publicai" V0 = "v0" diff --git a/litellm/utils.py b/litellm/utils.py index 7cac830b2c27..8790feadae06 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6729,6 +6729,11 @@ def validate_environment( # noqa: PLR0915 keys_in_environment = True else: missing_keys.append("DASHSCOPE_API_KEY") + elif custom_llm_provider == "modelscope": + if "MODELSCOPE_API_KEY" in os.environ: + keys_in_environment = True + else: + missing_keys.append("MODELSCOPE_API_KEY") elif custom_llm_provider == "moonshot": if "MOONSHOT_API_KEY" in os.environ: keys_in_environment = True @@ -8400,6 +8405,7 @@ def _build_provider_config_map() -> dict[LlmProviders, tuple[Callable, bool]]: LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), LlmProviders.DASHSCOPE: (lambda: litellm.DashScopeChatConfig(), False), + LlmProviders.MODELSCOPE: (lambda: litellm.ModelScopeChatConfig(), False), LlmProviders.MOONSHOT: (lambda: litellm.MoonshotChatConfig(), False), LlmProviders.DOCKER_MODEL_RUNNER: ( lambda: litellm.DockerModelRunnerChatConfig(), diff --git a/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py new file mode 100644 index 000000000000..7e2339221893 --- /dev/null +++ b/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py @@ -0,0 +1,114 @@ +""" +Unit tests for ModelScope configuration. + +These tests validate the DashScopeConfig class which extends OpenAIGPTConfig. +ModelScope is an OpenAI-compatible provider with minor customizations. +""" + +import os +import sys + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +import pytest + +import litellm +from litellm import completion +from litellm.llms.modelscope.chat.transformation import ModelScopeChatConfig + + +class TestModelScopeConfig: + """Test class for ModelScope functionality""" + + def test_default_api_base(self): + """Test that default API base is used when none is provided""" + config = ModelScopeChatConfig() + headers = {} + api_key = "fake-modelscope-key" + + # Call validate_environment without specifying api_base + result = config.validate_environment( + headers=headers, + model="Qwen/Qwen3-8B", + messages=[{"role": "user", "content": "Hey"}], + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base=None, # Not providing api_base + ) + + # Verify headers are still set correctly + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + + # We can't directly test the api_base value here since validate_environment + # only returns the headers, but we can verify it doesn't raise an exception + # which would happen if api_base handling was incorrect + + @pytest.mark.respx() + def test_modelscope_completion_mock(self, respx_mock): + """ + Mock test for ModelScope completion using the model format from docs. + This test mocks the actual HTTP request to test the integration properly. + """ + + litellm.disable_aiohttp_transport = ( + True # since this uses respx, we need to set use_aiohttp_transport to False + ) + + # Set up environment variables for the test + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + model = "Qwen/Qwen3-8B" + model_name = "Qwen3-8B" # The actual model name without provider prefix + + # Mock the HTTP request to the ModelScope API + respx_mock.post(f"{api_base}/chat/completions").respond( + json={ + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677652288, + "model": model_name, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": '```python\nprint("Hey from LiteLLM!")\n```\n\nThis simple Python code prints a greeting message from LiteLLM.', + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 9, + "completion_tokens": 12, + "total_tokens": 21, + }, + }, + status_code=200, + ) + + # Make the actual API call through LiteLLM + response = completion( + model=model, + messages=[ + {"role": "user", "content": "write code for saying hey from LiteLLM"} + ], + api_key=api_key, + api_base=api_base, + ) + + # Verify response structure + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + assert hasattr(response.choices[0], "message") + assert hasattr(response.choices[0].message, "content") + assert response.choices[0].message.content is not None + + # Check for specific content in the response + assert "```python" in response.choices[0].message.content + assert "Hey from LiteLLM" in response.choices[0].message.content + From f6ce4e16f8e7e563127f2f1807f3b9f5bbcb4c26 Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Thu, 14 May 2026 14:59:41 +0800 Subject: [PATCH 2/7] add modelscope api support --- litellm/__init__.py | 39 +++------ litellm/_lazy_imports_registry.py | 5 ++ litellm/constants.py | 84 +++++++++++++------ .../test_modelscope_chat_transformation.py | 2 +- 4 files changed, 78 insertions(+), 52 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index 405a83eea923..e0fb010c1d46 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -873,9 +873,9 @@ def add_known_models(model_cost_map: Optional[Dict] = None): elif value.get("litellm_provider") == "heroku": heroku_models.add(key) elif value.get("litellm_provider") == "dashscope": - dashscope_models.append(key) + dashscope_models.add(key) elif value.get("litellm_provider") == "modelscope": - modelscope_models.append(key) + modelscope_models.add(key) elif value.get("litellm_provider") == "moonshot": moonshot_models.add(key) elif value.get("litellm_provider") == "publicai": @@ -1239,30 +1239,14 @@ def add_known_models(model_cost_map: Optional[Dict] = None): # OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access from .llms.xai.common_utils import XAIModelInfo -from .llms.azure.chat.gpt_transformation import AzureOpenAIConfig -from .llms.azure.completion.transformation import AzureOpenAITextConfig -from .llms.hosted_vllm.chat.transformation import HostedVLLMChatConfig -from .llms.llamafile.chat.transformation import LlamafileChatConfig -from .llms.litellm_proxy.chat.transformation import LiteLLMProxyChatConfig -from .llms.vllm.completion.transformation import VLLMConfig -from .llms.deepseek.chat.transformation import DeepSeekChatConfig -from .llms.lm_studio.chat.transformation import LMStudioChatConfig -from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig -from .llms.nscale.chat.transformation import NscaleConfig -from .llms.perplexity.chat.transformation import PerplexityChatConfig -from .llms.azure.chat.o_series_transformation import AzureOpenAIO1Config -from .llms.watsonx.completion.transformation import IBMWatsonXAIConfig -from .llms.watsonx.chat.transformation import IBMWatsonXChatConfig -from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig -from .llms.github_copilot.chat.transformation import GithubCopilotConfig -from .llms.nebius.chat.transformation import NebiusConfig -from .llms.dashscope.chat.transformation import DashScopeChatConfig -from .llms.modelscope.chat.transformation import ModelScopeChatConfig -from .llms.moonshot.chat.transformation import MoonshotChatConfig -from .llms.v0.chat.transformation import V0ChatConfig -from .llms.morph.chat.transformation import MorphChatConfig -from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig -from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig +# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json) +# All remaining configs are now lazy loaded - see _lazy_imports_registry.py + +# Import LlmProviders here (before main import) because it's imported during import time +# in multiple places including openai.py (via main import) +from litellm.types.utils import LlmProviders + +## Lazy loading this is not straightforward, will leave it here for now. from .main import * # type: ignore from .compression import compress # type: ignore[no-redef] @@ -1958,6 +1942,9 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None: from .llms.dashscope.rerank.transformation import ( DashScopeRerankConfig as DashScopeRerankConfig, ) + from .llms.modelscope.chat.transformation import ( + ModelScopeChatConfig as ModelScopeChatConfig, + ) from .llms.moonshot.chat.transformation import ( MoonshotChatConfig as MoonshotChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index bdc3289b87c9..91e95b2d790c 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -304,6 +304,7 @@ "GigaChatConfig", "GigaChatEmbeddingConfig", "DashScopeChatConfig", + "ModelScopeChatConfig", "MoonshotChatConfig", "DockerModelRunnerChatConfig", "V0ChatConfig", @@ -1150,6 +1151,10 @@ ".llms.dashscope.chat.transformation", "DashScopeChatConfig", ), + "ModelScopeChatConfig": ( + ".llms.modelscope.chat.transformation", + "ModelScopeChatConfig", + ), "MoonshotChatConfig": (".llms.moonshot.chat.transformation", "MoonshotChatConfig"), "DockerModelRunnerChatConfig": ( ".llms.docker_model_runner.chat.transformation", diff --git a/litellm/constants.py b/litellm/constants.py index 3a6a6c8a31ba..3c439f9a36db 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1085,33 +1085,45 @@ ] ) +WANDB_MODELS: set = set( + [ + # openai models + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + # zai-org models + "zai-org/GLM-4.5", + # Qwen models + "Qwen/Qwen3-235B-A22B-Instruct-2507", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + # moonshotai + "moonshotai/Kimi-K2-Instruct", + "moonshotai/Kimi-K2.5", + # MiniMaxAI + "MiniMaxAI/MiniMax-M2.5", + # meta models + "meta-llama/Llama-3.1-8B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + "meta-llama/Llama-4-Scout-17B-16E-Instruct", + # deepseek-ai + "deepseek-ai/DeepSeek-V3.1", + "deepseek-ai/DeepSeek-R1-0528", + "deepseek-ai/DeepSeek-V3-0324", + # microsoft + "microsoft/Phi-4-mini-instruct", + ] +) + modelscope_models: List = [ + # LLM-Research models "LLM-Research/c4ai-command-r-plus-08-2024", + "LLM-Research/Llama-4-Scout-17B-16E-Instruct", + "LLM-Research/Llama-4-Maverick-17B-128E-Instruct", + # Mistral models "mistralai/Mistral-Small-Instruct-2409", "mistralai/Ministral-8B-Instruct-2410", "mistralai/Mistral-Large-Instruct-2407", - "Qwen/Qwen2.5-Coder-32B-Instruct", - "Qwen/Qwen2.5-Coder-14B-Instruct", - "Qwen/Qwen2.5-Coder-7B-Instruct", - "Qwen/Qwen2.5-72B-Instruct", - "Qwen/Qwen2.5-32B-Instruct", - "Qwen/Qwen2.5-14B-Instruct", - "Qwen/Qwen2.5-7B-Instruct", - "Qwen/QwQ-32B-Preview", - "opencompass/CompassJudger-1-32B-Instruct", - "Qwen/QVQ-72B-Preview", - "Qwen/Qwen2-VL-7B-Instruct", - "Qwen/Qwen2.5-14B-Instruct-1M", - "Qwen/Qwen2.5-7B-Instruct-1M", - "Qwen/Qwen2.5-VL-3B-Instruct", - "Qwen/Qwen2.5-VL-7B-Instruct", - "Qwen/Qwen2.5-VL-72B-Instruct", - "deepseek-ai/DeepSeek-V3", - "Qwen/QwQ-32B", - "XGenerationLab/XiYanSQL-QwenCoder-32B-2412", - "Qwen/Qwen2.5-VL-32B-Instruct", - "LLM-Research/Llama-4-Scout-17B-16E-Instruct", - "LLM-Research/Llama-4-Maverick-17B-128E-Instruct", + # Qwen3 series "Qwen/Qwen3-0.6B", "Qwen/Qwen3-1.7B", "Qwen/Qwen3-4B", @@ -1120,14 +1132,36 @@ "Qwen/Qwen3-30B-A3B", "Qwen/Qwen3-32B", "Qwen/Qwen3-235B-A22B", - "deepseek-ai/DeepSeek-R1-0528", - "MiniMax/MiniMax-M1-80k", "Qwen/Qwen3-235B-A22B-Instruct-2507", "Qwen/Qwen3-Coder-480B-A35B-Instruct", "Qwen/Qwen3-235B-A22B-Thinking-2507", - "ZhipuAI/GLM-4.5", "Qwen/Qwen3-30B-A3B-Thinking-2507", "Qwen/Qwen3-Coder-30B-A3B-Instruct", + # Qwen3.5 series (new) + "Qwen/Qwen3.5-0.6B", + "Qwen/Qwen3.5-1.7B", + "Qwen/Qwen3.5-3B", + "Qwen/Qwen3.5-7B", + "Qwen/Qwen3.5-14B", + "Qwen/Qwen3.5-30B-A3B", + "Qwen/Qwen3.5-32B", + "Qwen/Qwen3.5-235B-A22B", + "Qwen/Qwen3.5-235B-A22B-Instruct-2507", + "Qwen/Qwen3.5-Coder-480B-A35B-Instruct", + "Qwen/Qwen3.5-235B-A22B-Thinking-2507", + "Qwen/Qwen3.5-30B-A3B-Thinking-2507", + "Qwen/Qwen3.5-Coder-30B-A3B-Instruct", + # Other models + "Qwen/QwQ-32B-Preview", + "opencompass/CompassJudger-1-32B-Instruct", + "Qwen/QVQ-72B-Preview", + "Qwen/Qwen2-VL-7B-Instruct", + "deepseek-ai/DeepSeek-V3", + "Qwen/QwQ-32B", + "XGenerationLab/XiYanSQL-QwenCoder-32B-2412", + "deepseek-ai/DeepSeek-R1-0528", + "MiniMax/MiniMax-M1-80k", + "ZhipuAI/GLM-4.5", ] nebius_embedding_models: List = [ diff --git a/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py index 7e2339221893..c868eaaacaa4 100644 --- a/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py +++ b/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py @@ -61,7 +61,7 @@ def test_modelscope_completion_mock(self, respx_mock): # Set up environment variables for the test api_key = "fake-modelscope-key" api_base = "https://api-inference.modelscope.cn/v1" - model = "Qwen/Qwen3-8B" + model = "modelscope/Qwen/Qwen3-8B" # Use modelscope/ prefix to specify provider model_name = "Qwen3-8B" # The actual model name without provider prefix # Mock the HTTP request to the ModelScope API From 84bc74a4fd540053f34731a5fefdd8c9d1d1a7f2 Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Fri, 15 May 2026 09:49:35 +0800 Subject: [PATCH 3/7] update modelscope model list --- docs/my-website/docs/providers/modelscope.md | 242 ------------------- litellm/constants.py | 66 +++-- 2 files changed, 28 insertions(+), 280 deletions(-) delete mode 100644 docs/my-website/docs/providers/modelscope.md diff --git a/docs/my-website/docs/providers/modelscope.md b/docs/my-website/docs/providers/modelscope.md deleted file mode 100644 index b88041ae176e..000000000000 --- a/docs/my-website/docs/providers/modelscope.md +++ /dev/null @@ -1,242 +0,0 @@ -# ModelScope -LiteLLM supports running inference across multiple services for models hosted on the ModelScope Hub. - -## Supported Models - -### Serverless Inference Providers -You can check available models for an inference provider by going to [modelscope.cn/models](https://modelscope.cn/models), clicking the "API-Inference" and the "Other" filter tab, and selecting your desired provider. - -For example, you can find all Qwen3 series models [here](https://modelscope.cn/models?filter=inference_type&model_type=qwen3&page=1&tabKey=other). - - -### Dedicated Inference Endpoints -Refer to the [Inference Endpoints catalog](https://modelscope.cn/models?filter=inference_type&page=1&tabKey=task) for a list of available models. - -## Usage - - -### Authentication -With a single ModelScope token, you can access inference through multiple providers. Your calls are routed through ModelScope and the usage is free. -However, please ensure you bind your Alibaba Cloud account before use. For details, refer to the following two links. -- [API-Infereference](https://modelscope.cn/docs/model-service/API-Inference/intro) -- [Binding Alibaba Cloud Account](https://modelscope.cn/docs/accounts/aliyun-binding-and-authorization) - -Simply set the `MODELSCOPE_TOKEN` environment variable with your ModelScope token, you can create one here: https://modelscope.cn/my/myaccesstoken. - -```bash -export MODELSCOPE_TOKEN="123xxxxxx" -``` -or alternatively, you can pass your ModelScope token as a parameter: -```python -completion(..., api_key="123xxxxxx") -``` - -### Getting Started - -To use a ModelScope model, specify the model you want to use in the following format: -``` -// -``` -Where `/` is the ModelScope model ID. - -Examples: - -```python -# Run Llama-4-Scout-17B-16E-Instruct inference through LLM-Research -completion(model="modelscope/LLM-Research/Llama-4-Scout-17B-16E-Instruct",...) - -# Run DeepSeek-R1 inference through DeepSeek -completion(model="modelscope/deepseek-ai/DeepSeek-R1-0528",...) - -# Run Qwen3-8B inference through Qwen -completion(model="modelscope/Qwen/Qwen3-8B",...) -``` - - -### Basic Completion -Here's an example of chat completion using the Qwen3-8B model through Qwen: - -```python -import os -from litellm import completion - -os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" - -response = completion( - model="modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct", - messages=[ - { - "role": "user", - "content": "How many r's are in the word 'strawberry'?", - } - ], -) -print(response) -``` - -### Streaming -Now, let's see what a streaming request looks like. - -```python -import os -from litellm import completion - -os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" - -response = completion( - model="modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct", - messages=[ - { - "role": "user", - "content": "How many r's are in the word `strawberry`?", - - } - ], - stream=True, -) - -for chunk in response: - print(chunk) -``` - -### Image Input -You can also pass images when the model supports it. Here is an example using [Qwen/Qwen2.5-VL-72B-Instruct](https://modelscope.cn/models/Qwen/Qwen2.5-VL-72B-Instruct) model. - -```python -from litellm import completion - -# Set your ModelScope Token -os.environ["MODELSCOPE_TOKEN"] = "123xxxxxx" - -messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What's in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://modelscope.oss-cn-beijing.aliyuncs.com/demo/images/audrey_hepburn.jpg" - } - } - ] - } - ] - -response = completion( - model="modelscope/Qwen/Qwen2.5-VL-72B-Instruct", - messages=messages, -) -print(response.choices[0]) -``` - -## Model Deployment -SwingDeploy deployment service is a one-stop model deployment solution launched by ModelScope, aiming to provide developers with end-to-end services from model selection to cloud deployment. Through standardized deployment processes and cloud resource adaptation capabilities, users can quickly deploy the rich models of the Moda community (in multiple fields such as voice, video, and NLP) to the target cloud environment, achieving efficient implementation of model inference services. You can refer to [SwingDeploy](https://modelscope.cn/docs/model-service/deployment/intro) for more details. - -## LiteLLM Proxy Server with ModelScope models -You can set up a [LiteLLM Proxy Server](https://docs.litellm.ai/#litellm-proxy-server-llm-gateway) to serve ModelScope models through any of the supported Inference Providers. Here's how to do it: - -### Step 1. Setup the config file - -In this case, we are configuring a proxy to serve `Qwen3-Coder-480B-A35B-Instruct` from ModelScope. - -```yaml -model_list: - - model_name: my-model - litellm_params: - model: modelscope/Qwen/Qwen3-Coder-480B-A35B-Instruct - api_key: os.environ/MODELSCOPE_TOKEN # ensure you have `MODELSCOPE_TOKEN` in your .env -``` - -### Step 2. Start the server -```bash -litellm --config /path/to/config.yaml -``` - -### Step 3. Make a request to the server - - - -```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "my-model", - "messages": [ - { - "role": "user", - "content": "Hello, how are you?" - } - ] -}' -``` - - -## Usage with LiteLLM Proxy Server - -Here's how to call a ModelScope model with the LiteLLM Proxy Server - -1. Modify the config.yaml - - ```yaml showLineNumbers - model_list: - - model_name: my-model - litellm_params: - model: modelscope// # add modelscope/ prefix to route as ModelScope provider - api_key: api-key # api key to send your model - ``` - - -2. Start the proxy - - ```bash - $ litellm --config /path/to/config.yaml - ``` - -3. Send Request to LiteLLM Proxy Server - - - - - - ```python showLineNumbers - import openai - client = openai.OpenAI( - api_key="123xxxx", # pass litellm proxy key, if you're using virtual keys - base_url="http://0.0.0.0:4000" # litellm-proxy-base url - ) - - response = client.chat.completions.create( - model="my-model", - messages = [ - { - "role": "user", - "content": "what llm are you" - } - ], - ) - - print(response) - ``` - - - - - ```shell - curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer 1234xxxxx' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "my-model", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ] - }' - ``` - - - - diff --git a/litellm/constants.py b/litellm/constants.py index 3c439f9a36db..cb8414870ad2 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1085,6 +1085,12 @@ ] ) +nebius_embedding_models: List = [ + "BAAI/bge-en-icl", + "BAAI/bge-multilingual-gemma2", + "intfloat/e5-mistral-7b-instruct", +] + WANDB_MODELS: set = set( [ # openai models @@ -1115,15 +1121,7 @@ ) modelscope_models: List = [ - # LLM-Research models - "LLM-Research/c4ai-command-r-plus-08-2024", - "LLM-Research/Llama-4-Scout-17B-16E-Instruct", - "LLM-Research/Llama-4-Maverick-17B-128E-Instruct", - # Mistral models - "mistralai/Mistral-Small-Instruct-2409", - "mistralai/Ministral-8B-Instruct-2410", - "mistralai/Mistral-Large-Instruct-2407", - # Qwen3 series + # Qwen series models "Qwen/Qwen3-0.6B", "Qwen/Qwen3-1.7B", "Qwen/Qwen3-4B", @@ -1133,41 +1131,33 @@ "Qwen/Qwen3-32B", "Qwen/Qwen3-235B-A22B", "Qwen/Qwen3-235B-A22B-Instruct-2507", - "Qwen/Qwen3-Coder-480B-A35B-Instruct", "Qwen/Qwen3-235B-A22B-Thinking-2507", "Qwen/Qwen3-30B-A3B-Thinking-2507", "Qwen/Qwen3-Coder-30B-A3B-Instruct", - # Qwen3.5 series (new) - "Qwen/Qwen3.5-0.6B", - "Qwen/Qwen3.5-1.7B", - "Qwen/Qwen3.5-3B", - "Qwen/Qwen3.5-7B", - "Qwen/Qwen3.5-14B", - "Qwen/Qwen3.5-30B-A3B", - "Qwen/Qwen3.5-32B", - "Qwen/Qwen3.5-235B-A22B", - "Qwen/Qwen3.5-235B-A22B-Instruct-2507", - "Qwen/Qwen3.5-Coder-480B-A35B-Instruct", - "Qwen/Qwen3.5-235B-A22B-Thinking-2507", - "Qwen/Qwen3.5-30B-A3B-Thinking-2507", - "Qwen/Qwen3.5-Coder-30B-A3B-Instruct", - # Other models + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Thinking", + "Qwen/Qwen3-VL-235B-A22B-Instruct", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-8B-Thinking", + "Qwen/Qwen3.5-122B-A10B", + "Qwen/Qwen3.5-27B", + "Qwen/Qwen3.5-35B-A3B", + "Qwen/Qwen3.5-397B-A17B", + "Qwen/QwQ-32B", "Qwen/QwQ-32B-Preview", - "opencompass/CompassJudger-1-32B-Instruct", "Qwen/QVQ-72B-Preview", - "Qwen/Qwen2-VL-7B-Instruct", - "deepseek-ai/DeepSeek-V3", - "Qwen/QwQ-32B", - "XGenerationLab/XiYanSQL-QwenCoder-32B-2412", + "Qwen/Qwen-Image-Edit", + # DeepSeek series models "deepseek-ai/DeepSeek-R1-0528", - "MiniMax/MiniMax-M1-80k", - "ZhipuAI/GLM-4.5", -] - -nebius_embedding_models: List = [ - "BAAI/bge-en-icl", - "BAAI/bge-multilingual-gemma2", - "intfloat/e5-mistral-7b-instruct", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "deepseek-ai/DeepSeek-V3.2", + "deepseek-ai/DeepSeek-V4-Flash", ] BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ From 1b9acac18a3857e7f383f278d052291935cb1a90 Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Tue, 19 May 2026 09:48:40 +0800 Subject: [PATCH 4/7] add image-genetation support --- .../get_llm_provider_logic.py | 2 +- .../llms/modelscope/chat/transformation.py | 6 +- .../modelscope/image_generation/__init__.py | 31 ++ .../image_generation/transformation.py | 246 ++++++++++ litellm/utils.py | 6 + .../test_modelscope_chat_transformation.py | 2 +- ...est_modelscope_image_gen_transformation.py | 460 ++++++++++++++++++ 7 files changed, 749 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/modelscope/image_generation/__init__.py create mode 100644 litellm/llms/modelscope/image_generation/transformation.py rename tests/test_litellm/llms/modelscope/{ => chat}/test_modelscope_chat_transformation.py (97%) create mode 100644 tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 7b9e575e030b..ef13215495ef 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -334,7 +334,7 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "dashscope-intl.aliyuncs.com/compatible-mode/v1": custom_llm_provider = "dashscope" dynamic_api_key = get_secret_str("DASHSCOPE_API_KEY") - elif endpoint == "api-inference.modelscope.cn/v1": + elif endpoint == "https://api-inference.modelscope.cn/v1": custom_llm_provider = "modelscope" dynamic_api_key = get_secret_str("MODELSCOPE_API_KEY") elif endpoint == "api.moonshot.ai/v1": diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py index b657bc8950f3..257959685e8a 100644 --- a/litellm/llms/modelscope/chat/transformation.py +++ b/litellm/llms/modelscope/chat/transformation.py @@ -14,6 +14,8 @@ class ModelScopeChatConfig(OpenAIGPTConfig): + DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" + @overload def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: Literal[True] @@ -49,7 +51,7 @@ def _get_openai_compatible_provider_info( api_base = ( api_base or get_secret_str("MODELSCOPE_API_BASE") - or "https://api-inference.modelscope.cn/v1" + or self.DEFAULT_BASE_URL ) # type: ignore dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY") return api_base, dynamic_api_key @@ -67,7 +69,7 @@ def get_complete_url( If api_base is not provided, use the default ModelScope /chat/completions endpoint. """ if not api_base: - api_base = "https://api-inference.modelscope.cn/v1" + api_base = self.DEFAULT_BASE_URL if not api_base.endswith("/chat/completions"): api_base = f"{api_base}/chat/completions" diff --git a/litellm/llms/modelscope/image_generation/__init__.py b/litellm/llms/modelscope/image_generation/__init__.py new file mode 100644 index 000000000000..8b28ea962ce9 --- /dev/null +++ b/litellm/llms/modelscope/image_generation/__init__.py @@ -0,0 +1,31 @@ +""" +ModelScope Image Generation Module + +Factory function for getting the appropriate config class. +""" + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) + +from .transformation import ModelScopeImageGenerationConfig + +__all__ = [ + "ModelScopeImageGenerationConfig", + "get_modelscope_image_generation_config", +] + + +def get_modelscope_image_generation_config( + model: str, +) -> BaseImageGenerationConfig: + """ + Get the ModelScope config for image generation. + + Args: + model: The model name (e.g., "modelscope/Qwen/Qwen-Image-Edit") + + Returns: + BaseImageGenerationConfig instance for ModelScope + """ + return ModelScopeImageGenerationConfig() diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py new file mode 100644 index 000000000000..2716a8057ef3 --- /dev/null +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -0,0 +1,246 @@ +""" +ModelScope Image Generation Config + +Handles transformation between OpenAI-compatible format and ModelScope API format. + +API Reference: https://modelscope.cn/docs/model-service/API-Inference/intro +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Union + +import httpx + +from litellm.llms.base_llm.image_generation.transformation import ( + BaseImageGenerationConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIImageGenerationOptionalParams, +) +from litellm.types.utils import ImageObject, ImageResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): + """ + Configuration for ModelScope image generation. + + Supports text-to-image models like: + - Qwen/Qwen-Image-Edit + - And other ModelScope-hosted image generation models + """ + + DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + """ + Return list of OpenAI params supported by ModelScope. + + ModelScope supports standard OpenAI image generation parameters. + """ + return [ + "n", # Number of images to generate + "size", # Size of the generated images + "response_format", # url or b64_json + "user", # User identifier + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """ + Map OpenAI parameters to ModelScope parameters. + + ModelScope uses the same parameter names as OpenAI. + """ + supported_params = self.get_supported_openai_params(model) + if drop_params: + non_default_params = { + k: v for k, v in non_default_params.items() if k in supported_params + } + optional_params.update(non_default_params) + return optional_params + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the ModelScope image generation API request. + """ + base_url: str = ( + api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL + ) + base_url = base_url.rstrip("/") + + # Return the images endpoint + return f"{base_url}/images/generations" + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and set up headers for ModelScope. + """ + final_api_key: Optional[str] = api_key or get_secret_str("MODELSCOPE_API_KEY") + + if not final_api_key: + raise ValueError( + "MODELSCOPE_API_KEY is not set. " + "Please set it via environment variable or pass api_key parameter." + ) + + default_headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {final_api_key}", + } + + headers = {**headers, **default_headers} + return headers + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform OpenAI-style request to ModelScope request format. + + ModelScope uses the same format as OpenAI for image generation. + """ + # Build the request body (same as OpenAI) + request_data: dict = { + "model": model, + "prompt": prompt, + } + + # Add optional params + for key, value in optional_params.items(): + if key.startswith("_"): + continue + request_data[key] = value + + return request_data + + def transform_image_generation_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ImageResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ImageResponse: + """ + Transform ModelScope response to OpenAI-compatible ImageResponse. + + ModelScope returns the same format as OpenAI: + {"created": timestamp, "data": [{"url": "..."}]} + """ + try: + response_data = raw_response.json() + except Exception as e: + raise self.get_error_class( + error_message=f"Error parsing ModelScope response: {e}", + status_code=raw_response.status_code, + headers=raw_response.headers, + model=model, + ) + + # Check for errors in response + if "error" in response_data: + error_msg = response_data["error"].get( + "message", str(response_data["error"]) + ) + raise self.get_error_class( + error_message=f"ModelScope error: {error_msg}", + status_code=raw_response.status_code, + headers=raw_response.headers, + model=model, + ) + + # Extract images from response + data_list = response_data.get("data", []) + if not model_response.data: + model_response.data = [] + + for item in data_list: + image_obj = ImageObject( + url=item.get("url"), + b64_json=item.get("b64_json"), + revised_prompt=item.get("revised_prompt"), + ) + model_response.data.append(image_obj) + + return model_response + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Union[dict, httpx.Headers], + model: Optional[str] = None, + ) -> Exception: + """Return the appropriate error class for ModelScope.""" + from litellm.exceptions import ( + AuthenticationError, + BadRequestError, + InternalServerError, + ) + + if status_code == 400: + return BadRequestError( + message=error_message, + model=model or "", + llm_provider="modelscope", + ) + elif status_code == 401: + return AuthenticationError( + message=error_message, + model=model or "", + llm_provider="modelscope", + ) + elif status_code >= 500: + return InternalServerError( + message=error_message, + model=model or "", + llm_provider="modelscope", + ) + else: + return BadRequestError( + message=error_message, + model=model or "", + llm_provider="modelscope", + ) diff --git a/litellm/utils.py b/litellm/utils.py index 8790feadae06..0c89eeec325b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9298,6 +9298,12 @@ def get_provider_image_generation_config( ) return get_dashscope_image_generation_config(model) + elif LlmProviders.MODELSCOPE == provider: + from litellm.llms.modelscope.image_generation import ( + get_modelscope_image_generation_config, + ) + + return get_modelscope_image_generation_config(model) return None @staticmethod diff --git a/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py similarity index 97% rename from tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py rename to tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py index c868eaaacaa4..89c521533a11 100644 --- a/tests/test_litellm/llms/modelscope/test_modelscope_chat_transformation.py +++ b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py @@ -1,7 +1,7 @@ """ Unit tests for ModelScope configuration. -These tests validate the DashScopeConfig class which extends OpenAIGPTConfig. +These tests validate the ModelScopeChatConfig class which extends OpenAIGPTConfig. ModelScope is an OpenAI-compatible provider with minor customizations. """ diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py new file mode 100644 index 000000000000..69d50237f72c --- /dev/null +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -0,0 +1,460 @@ +""" +Unit tests for ModelScope image generation configuration. + +These tests validate the ModelScopeImageGenerationConfig class which handles +transformation between OpenAI-compatible format and ModelScope API format. +""" + +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +sys.path.insert( + 0, os.path.abspath("../../../../..") +) # Adds the parent directory to the system path + +from litellm.llms.modelscope.image_generation.transformation import ( + ModelScopeImageGenerationConfig, +) +from litellm.types.utils import ImageResponse + + +class TestModelScopeImageGenerationTransformation: + def setup_method(self): + """Set up test fixtures before each test method.""" + self.config = ModelScopeImageGenerationConfig() + self.model = "modelscope/Qwen/Qwen-Image-Edit" + self.logging_obj = MagicMock() + + def test_get_supported_openai_params(self): + """Test that get_supported_openai_params returns correct parameters.""" + supported_params = self.config.get_supported_openai_params(self.model) + + assert "n" in supported_params + assert "size" in supported_params + assert "response_format" in supported_params + assert "user" in supported_params + + def test_map_openai_params(self): + """Test that map_openai_params correctly passes through parameters.""" + non_default_params = { + "n": 2, + "size": "1024x1024", + "response_format": "url", + } + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False, + ) + + assert result["n"] == 2 + assert result["size"] == "1024x1024" + assert result["response_format"] == "url" + + def test_map_openai_params_with_user(self): + """Test that map_openai_params correctly passes through user parameter.""" + non_default_params = {"user": "test-user-123"} + optional_params = {} + + result = self.config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=self.model, + drop_params=False, + ) + + assert result["user"] == "test-user-123" + + def test_get_complete_url_default(self): + """Test that get_complete_url returns default ModelScope URL.""" + result = self.config.get_complete_url( + api_base=None, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == "https://api-inference.modelscope.cn/v1/images/generations" + + def test_get_complete_url_with_custom_base(self): + """Test that get_complete_url uses custom api_base.""" + custom_base = "https://custom.modelscope.cn/v1" + + result = self.config.get_complete_url( + api_base=custom_base, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == f"{custom_base}/images/generations" + + def test_get_complete_url_with_trailing_slash(self): + """Test that get_complete_url strips trailing slashes from base.""" + custom_base = "https://custom.modelscope.cn/v1/" + + result = self.config.get_complete_url( + api_base=custom_base, + api_key="test_key", + model=self.model, + optional_params={}, + litellm_params={}, + ) + + assert result == "https://custom.modelscope.cn/v1/images/generations" + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_with_api_key(self, mock_get_secret): + """Test that validate_environment correctly sets authorization header.""" + headers = {} + api_key = "test_api_key" + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=api_key, + ) + + assert result["Authorization"] == f"Bearer {api_key}" + assert result["Content-Type"] == "application/json" + mock_get_secret.assert_not_called() + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_with_secret_key(self, mock_get_secret): + """Test that validate_environment uses secret API key when api_key is None.""" + mock_get_secret.return_value = "secret_api_key" + headers = {} + + result = self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + assert result["Authorization"] == "Bearer secret_api_key" + mock_get_secret.assert_called_once_with("MODELSCOPE_API_KEY") + + @patch("litellm.llms.modelscope.image_generation.transformation.get_secret_str") + def test_validate_environment_no_api_key(self, mock_get_secret): + """Test that validate_environment raises error when no API key is available.""" + mock_get_secret.return_value = None + headers = {} + + with pytest.raises(ValueError) as exc_info: + self.config.validate_environment( + headers=headers, + model=self.model, + messages=[], + optional_params={}, + litellm_params={}, + api_key=None, + ) + + assert "MODELSCOPE_API_KEY is not set" in str(exc_info.value) + + def test_transform_image_generation_request_basic(self): + """Test that transform_image_generation_request creates correct request body.""" + prompt = "A beautiful sunset over mountains" + optional_params = {} + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["prompt"] == prompt + + def test_transform_image_generation_request_with_optional_params(self): + """Test that transform_image_generation_request includes optional params.""" + prompt = "A beautiful sunset" + optional_params = { + "n": 2, + "size": "1024x1024", + "response_format": "b64_json", + } + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["prompt"] == prompt + assert result["n"] == 2 + assert result["size"] == "1024x1024" + assert result["response_format"] == "b64_json" + + def test_transform_image_generation_request_ignores_internal_params(self): + """Test that transform_image_generation_request ignores params starting with _.""" + prompt = "A beautiful sunset" + optional_params = { + "n": 2, + "_internal_param": "should_be_ignored", + } + + result = self.config.transform_image_generation_request( + model=self.model, + prompt=prompt, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + assert result["model"] == self.model + assert result["n"] == 2 + assert "_internal_param" not in result + + def test_transform_image_generation_response_with_url_images(self): + """Test that transform_image_generation_response correctly extracts URL images.""" + response_data = { + "created": 1234567890, + "data": [ + {"url": "https://example.com/image1.png"}, + {"url": "https://example.com/image2.png"}, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 2 + assert result.data[0].url == "https://example.com/image1.png" + assert result.data[1].url == "https://example.com/image2.png" + + def test_transform_image_generation_response_with_b64_json(self): + """Test that transform_image_generation_response correctly extracts base64 images.""" + response_data = { + "created": 1234567890, + "data": [ + {"b64_json": "iVBORw0KGgoAAAANS"}, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert result.data[0].b64_json == "iVBORw0KGgoAAAANS" + assert result.data[0].url is None + + def test_transform_image_generation_response_with_revised_prompt(self): + """Test that transform_image_generation_response extracts revised_prompt.""" + response_data = { + "created": 1234567890, + "data": [ + { + "url": "https://example.com/image.png", + "revised_prompt": "A detailed description of a beautiful sunset", + }, + ], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 1 + assert ( + result.data[0].revised_prompt + == "A detailed description of a beautiful sunset" + ) + + def test_transform_image_generation_response_empty_data(self): + """Test that transform_image_generation_response handles empty data array.""" + response_data = { + "created": 1234567890, + "data": [], + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 200 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + result = self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert len(result.data) == 0 + + def test_transform_image_generation_response_error_handling(self): + """Test that transform_image_generation_response raises error on API error.""" + response_data = { + "error": { + "message": "Invalid prompt provided", + "type": "invalid_request_error", + } + } + + mock_response = MagicMock() + mock_response.json.return_value = response_data + mock_response.status_code = 400 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + with pytest.raises(Exception) as exc_info: + self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "ModelScope error" in str(exc_info.value) + assert "Invalid prompt provided" in str(exc_info.value) + + def test_transform_image_generation_response_json_error(self): + """Test that transform_image_generation_response raises error on invalid JSON.""" + import json + + mock_response = MagicMock() + mock_response.json.side_effect = json.JSONDecodeError("Invalid JSON", "", 0) + mock_response.status_code = 500 + mock_response.headers = {} + + model_response = ImageResponse(data=[]) + + with pytest.raises(Exception) as exc_info: + self.config.transform_image_generation_response( + model=self.model, + raw_response=mock_response, + model_response=model_response, + logging_obj=self.logging_obj, + request_data={}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert "Error parsing ModelScope response" in str(exc_info.value) + + def test_get_error_class_bad_request(self): + """Test that get_error_class returns BadRequestError for 400 status.""" + from litellm.exceptions import BadRequestError + + error = self.config.get_error_class( + error_message="Bad request", + status_code=400, + headers={"Content-Type": "application/json"}, + model=self.model, + ) + + assert isinstance(error, BadRequestError) + + def test_get_error_class_authentication_error(self): + """Test that get_error_class returns AuthenticationError for 401 status.""" + from litellm.exceptions import AuthenticationError + + error = self.config.get_error_class( + error_message="Invalid API key", + status_code=401, + headers={"Content-Type": "application/json"}, + model=self.model, + ) + + assert isinstance(error, AuthenticationError) + + def test_get_error_class_internal_server_error(self): + """Test that get_error_class returns InternalServerError for 500+ status.""" + from litellm.exceptions import InternalServerError + + error = self.config.get_error_class( + error_message="Internal server error", + status_code=500, + headers={"Content-Type": "application/json"}, + model=self.model, + ) + + assert isinstance(error, InternalServerError) + + def test_get_error_class_default(self): + """Test that get_error_class returns BadRequestError for other status codes.""" + from litellm.exceptions import BadRequestError + + error = self.config.get_error_class( + error_message="Some error", + status_code=404, + headers={"Content-Type": "application/json"}, + model=self.model, + ) + + assert isinstance(error, BadRequestError) From 76d880a93be39be92a4d987b82365af02c17a413 Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Tue, 19 May 2026 14:40:29 +0800 Subject: [PATCH 5/7] update test and multimodal --- litellm/constants.py | 92 ++--- .../llms/modelscope/chat/transformation.py | 26 +- .../image_generation/transformation.py | 22 +- .../test_modelscope_chat_transformation.py | 342 ++++++++++++++++-- ...est_modelscope_image_gen_transformation.py | 4 - 5 files changed, 390 insertions(+), 96 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index cb8414870ad2..8eba230baabe 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1085,11 +1085,13 @@ ] ) -nebius_embedding_models: List = [ - "BAAI/bge-en-icl", - "BAAI/bge-multilingual-gemma2", - "intfloat/e5-mistral-7b-instruct", -] +nebius_embedding_models: set = set( + [ + "BAAI/bge-en-icl", + "BAAI/bge-multilingual-gemma2", + "intfloat/e5-mistral-7b-instruct", + ] +) WANDB_MODELS: set = set( [ @@ -1120,45 +1122,47 @@ ] ) -modelscope_models: List = [ - # Qwen series models - "Qwen/Qwen3-0.6B", - "Qwen/Qwen3-1.7B", - "Qwen/Qwen3-4B", - "Qwen/Qwen3-8B", - "Qwen/Qwen3-14B", - "Qwen/Qwen3-30B-A3B", - "Qwen/Qwen3-32B", - "Qwen/Qwen3-235B-A22B", - "Qwen/Qwen3-235B-A22B-Instruct-2507", - "Qwen/Qwen3-235B-A22B-Thinking-2507", - "Qwen/Qwen3-30B-A3B-Thinking-2507", - "Qwen/Qwen3-Coder-30B-A3B-Instruct", - "Qwen/Qwen3-Coder-480B-A35B-Instruct", - "Qwen/Qwen3-Next-80B-A3B-Instruct", - "Qwen/Qwen3-Next-80B-A3B-Thinking", - "Qwen/Qwen3-VL-235B-A22B-Instruct", - "Qwen/Qwen3-VL-8B-Instruct", - "Qwen/Qwen3-VL-8B-Thinking", - "Qwen/Qwen3.5-122B-A10B", - "Qwen/Qwen3.5-27B", - "Qwen/Qwen3.5-35B-A3B", - "Qwen/Qwen3.5-397B-A17B", - "Qwen/QwQ-32B", - "Qwen/QwQ-32B-Preview", - "Qwen/QVQ-72B-Preview", - "Qwen/Qwen-Image-Edit", - # DeepSeek series models - "deepseek-ai/DeepSeek-R1-0528", - "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", - "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", - "deepseek-ai/DeepSeek-V3.2", - "deepseek-ai/DeepSeek-V4-Flash", -] +modelscope_models: set = set( + [ + # Qwen series models + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3-1.7B", + "Qwen/Qwen3-4B", + "Qwen/Qwen3-8B", + "Qwen/Qwen3-14B", + "Qwen/Qwen3-30B-A3B", + "Qwen/Qwen3-32B", + "Qwen/Qwen3-235B-A22B", + "Qwen/Qwen3-235B-A22B-Instruct-2507", + "Qwen/Qwen3-235B-A22B-Thinking-2507", + "Qwen/Qwen3-30B-A3B-Thinking-2507", + "Qwen/Qwen3-Coder-30B-A3B-Instruct", + "Qwen/Qwen3-Coder-480B-A35B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Instruct", + "Qwen/Qwen3-Next-80B-A3B-Thinking", + "Qwen/Qwen3-VL-235B-A22B-Instruct", + "Qwen/Qwen3-VL-8B-Instruct", + "Qwen/Qwen3-VL-8B-Thinking", + "Qwen/Qwen3.5-122B-A10B", + "Qwen/Qwen3.5-27B", + "Qwen/Qwen3.5-35B-A3B", + "Qwen/Qwen3.5-397B-A17B", + "Qwen/QwQ-32B", + "Qwen/QwQ-32B-Preview", + "Qwen/QVQ-72B-Preview", + "Qwen/Qwen-Image-Edit", + # DeepSeek series models + "deepseek-ai/DeepSeek-R1-0528", + "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", + "deepseek-ai/DeepSeek-R1-Distill-Llama-8B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-14B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B", + "deepseek-ai/DeepSeek-R1-Distill-Qwen-7B", + "deepseek-ai/DeepSeek-V3.2", + "deepseek-ai/DeepSeek-V4-Flash", + ] +) BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "cohere", diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py index 257959685e8a..9f26bee5f540 100644 --- a/litellm/llms/modelscope/chat/transformation.py +++ b/litellm/llms/modelscope/chat/transformation.py @@ -4,15 +4,20 @@ from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload -from litellm.litellm_core_utils.prompt_templates.common_utils import ( - handle_messages_with_content_list_to_str_conversion, -) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from ...openai.chat.gpt_transformation import OpenAIGPTConfig +def _has_non_text_content(message: AllMessageValues) -> bool: + """Check if a message has non-text content items (e.g. image_url).""" + content = message.get("content") + if not isinstance(content, list): + return False + return any(item.get("type") != "text" for item in content) + + class ModelScopeChatConfig(OpenAIGPTConfig): DEFAULT_BASE_URL: str = "https://api-inference.modelscope.cn/v1" @@ -33,9 +38,20 @@ def _transform_messages( self, messages: List[AllMessageValues], model: str, is_async: bool = False ) -> Union[List[AllMessageValues], Coroutine[Any, Any, List[AllMessageValues]]]: """ - ModelScope does not support content in list format. + Flatten text-only content lists to strings for ModelScope. + + Messages with non-text content (e.g. image_url for vision models) + are kept as lists so the parent class can normalize them properly. """ - messages = handle_messages_with_content_list_to_str_conversion(messages) + for message in messages: + if _has_non_text_content(message): + continue + content = message.get("content") + if isinstance(content, list): + message["content"] = "".join( + item.get("text") or "" for item in content + ) + if is_async: return super()._transform_messages( messages=messages, model=model, is_async=True diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py index 2716a8057ef3..6b848b20ee93 100644 --- a/litellm/llms/modelscope/image_generation/transformation.py +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -10,6 +10,7 @@ import httpx +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -176,7 +177,6 @@ def transform_image_generation_response( error_message=f"Error parsing ModelScope response: {e}", status_code=raw_response.status_code, headers=raw_response.headers, - model=model, ) # Check for errors in response @@ -188,7 +188,6 @@ def transform_image_generation_response( error_message=f"ModelScope error: {error_msg}", status_code=raw_response.status_code, headers=raw_response.headers, - model=model, ) # Extract images from response @@ -211,8 +210,7 @@ def get_error_class( error_message: str, status_code: int, headers: Union[dict, httpx.Headers], - model: Optional[str] = None, - ) -> Exception: + ) -> BaseLLMException: """Return the appropriate error class for ModelScope.""" from litellm.exceptions import ( AuthenticationError, @@ -221,26 +219,26 @@ def get_error_class( ) if status_code == 400: - return BadRequestError( + return BadRequestError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) elif status_code == 401: - return AuthenticationError( + return AuthenticationError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) elif status_code >= 500: - return InternalServerError( + return InternalServerError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) else: - return BadRequestError( + return BadRequestError( # type: ignore[return-value] message=error_message, - model=model or "", + model="", llm_provider="modelscope", ) diff --git a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py index 89c521533a11..2767deae1763 100644 --- a/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py +++ b/tests/test_litellm/llms/modelscope/chat/test_modelscope_chat_transformation.py @@ -5,6 +5,7 @@ ModelScope is an OpenAI-compatible provider with minor customizations. """ +import json import os import sys @@ -12,12 +13,18 @@ 0, os.path.abspath("../../../../..") ) # Adds the parent directory to the system path +from unittest.mock import patch + +import httpx import pytest +import respx import litellm from litellm import completion from litellm.llms.modelscope.chat.transformation import ModelScopeChatConfig +DEFAULT_MODEL = "Qwen/Qwen3.5-35B-A3B" + class TestModelScopeConfig: """Test class for ModelScope functionality""" @@ -28,55 +35,40 @@ def test_default_api_base(self): headers = {} api_key = "fake-modelscope-key" - # Call validate_environment without specifying api_base result = config.validate_environment( headers=headers, - model="Qwen/Qwen3-8B", + model=DEFAULT_MODEL, messages=[{"role": "user", "content": "Hey"}], optional_params={}, litellm_params={}, api_key=api_key, - api_base=None, # Not providing api_base + api_base=None, ) - # Verify headers are still set correctly assert result["Authorization"] == f"Bearer {api_key}" assert result["Content-Type"] == "application/json" - # We can't directly test the api_base value here since validate_environment - # only returns the headers, but we can verify it doesn't raise an exception - # which would happen if api_base handling was incorrect - @pytest.mark.respx() def test_modelscope_completion_mock(self, respx_mock): - """ - Mock test for ModelScope completion using the model format from docs. - This test mocks the actual HTTP request to test the integration properly. - """ + """Mock test for basic ModelScope completion.""" - litellm.disable_aiohttp_transport = ( - True # since this uses respx, we need to set use_aiohttp_transport to False - ) + litellm.disable_aiohttp_transport = True - # Set up environment variables for the test api_key = "fake-modelscope-key" api_base = "https://api-inference.modelscope.cn/v1" - model = "modelscope/Qwen/Qwen3-8B" # Use modelscope/ prefix to specify provider - model_name = "Qwen3-8B" # The actual model name without provider prefix - # Mock the HTTP request to the ModelScope API respx_mock.post(f"{api_base}/chat/completions").respond( json={ "id": "chatcmpl-123", "object": "chat.completion", "created": 1677652288, - "model": model_name, + "model": DEFAULT_MODEL, "choices": [ { "index": 0, "message": { "role": "assistant", - "content": '```python\nprint("Hey from LiteLLM!")\n```\n\nThis simple Python code prints a greeting message from LiteLLM.', + "content": '```python\nprint("Hey from LiteLLM!")\n```', }, "finish_reason": "stop", } @@ -90,9 +82,8 @@ def test_modelscope_completion_mock(self, respx_mock): status_code=200, ) - # Make the actual API call through LiteLLM response = completion( - model=model, + model=f"modelscope/{DEFAULT_MODEL}", messages=[ {"role": "user", "content": "write code for saying hey from LiteLLM"} ], @@ -100,15 +91,304 @@ def test_modelscope_completion_mock(self, respx_mock): api_base=api_base, ) - # Verify response structure assert response is not None - assert hasattr(response, "choices") - assert len(response.choices) > 0 - assert hasattr(response.choices[0], "message") - assert hasattr(response.choices[0].message, "content") assert response.choices[0].message.content is not None - - # Check for specific content in the response assert "```python" in response.choices[0].message.content - assert "Hey from LiteLLM" in response.choices[0].message.content + # ── _transform_messages tests ────────────────────────────────────── + + def test_transform_messages_flattens_text_content_list(self): + """Content lists containing only text items should be flattened to a string.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello world" + + def test_transform_messages_preserves_multimodal_content_list(self): + """Content lists with image_url should be preserved as lists for vision models.""" + config = ModelScopeChatConfig() + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": "https://example.com/img.png"}}, + ], + } + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert isinstance(result[0]["content"], list) + assert len(result[0]["content"]) == 2 + assert result[0]["content"][0]["type"] == "text" + assert result[0]["content"][1]["type"] == "image_url" + + def test_transform_messages_string_content_unchanged(self): + """Messages with string content should pass through unchanged.""" + config = ModelScopeChatConfig() + messages = [{"role": "user", "content": "Hello"}] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hello" + + def test_transform_messages_multi_turn(self): + """Multi-turn conversations should be handled correctly.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Tell me more"}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + assert result[2]["content"] == "Tell me more" + + def test_transform_messages_multimodal_multi_turn(self): + """Multi-turn with mixed text-only and multimodal messages.""" + config = ModelScopeChatConfig() + messages = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello!"}, + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image"}, + {"type": "image_url", "image_url": {"url": "https://example.com/photo.jpg"}}, + ], + }, + ] + + result = config._transform_messages(messages=messages, model=DEFAULT_MODEL) + + assert result[0]["content"] == "Hi" + assert result[1]["content"] == "Hello!" + # Multimodal message should keep list format + assert isinstance(result[2]["content"], list) + assert result[2]["content"][1]["type"] == "image_url" + + # ── get_complete_url tests ───────────────────────────────────────── + + def test_get_complete_url_default(self): + """Default api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base=None, + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_custom_base(self): + """Custom api_base should append /chat/completions.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://custom.modelscope.cn/v1", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://custom.modelscope.cn/v1/chat/completions" + + def test_get_complete_url_already_has_endpoint(self): + """api_base already ending in /chat/completions should not be doubled.""" + config = ModelScopeChatConfig() + + url = config.get_complete_url( + api_base="https://api-inference.modelscope.cn/v1/chat/completions", + api_key="fake-key", + model=DEFAULT_MODEL, + optional_params={}, + litellm_params={}, + ) + + assert url == "https://api-inference.modelscope.cn/v1/chat/completions" + assert url.count("/chat/completions") == 1 + + # ── _get_openai_compatible_provider_info tests ───────────────────── + + def test_get_provider_info_with_explicit_api_base(self): + """Explicit api_base and api_key should be returned as-is.""" + config = ModelScopeChatConfig() + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base="https://custom.example.com/v1", + api_key="my-key", + ) + + assert api_base == "https://custom.example.com/v1" + assert api_key == "my-key" + + def test_get_provider_info_default_fallback(self): + """When no api_base or env var is set, DEFAULT_BASE_URL should be used.""" + config = ModelScopeChatConfig() + + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("MODELSCOPE_API_BASE", None) + os.environ.pop("MODELSCOPE_API_KEY", None) + + api_base, api_key = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://api-inference.modelscope.cn/v1" + assert api_key is None + + def test_get_provider_info_env_var_fallback(self): + """MODELSCOPE_API_BASE env var should be used when api_base is not provided.""" + config = ModelScopeChatConfig() + + with patch.dict( + os.environ, + {"MODELSCOPE_API_BASE": "https://env.modelscope.cn/v1"}, + ): + api_base, _ = config._get_openai_compatible_provider_info( + api_base=None, + api_key=None, + ) + + assert api_base == "https://env.modelscope.cn/v1" + + # ── Mock HTTP tests ──────────────────────────────────────────────── + + @pytest.mark.respx() + def test_completion_with_text_content_list(self, respx_mock): + """Verify that text-only content list messages are flattened before sending.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-456", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Sure!"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello"}, + {"type": "text", "text": " world"}, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "Sure!" + + body = json.loads(captured_request["body"]) + assert isinstance(body["messages"][0]["content"], str) + assert body["messages"][0]["content"] == "Hello world" + + @pytest.mark.respx() + def test_completion_with_multimodal_messages(self, respx_mock): + """Verify that multimodal messages (text + image_url) are sent as content lists.""" + litellm.disable_aiohttp_transport = True + + api_key = "fake-modelscope-key" + api_base = "https://api-inference.modelscope.cn/v1" + captured_request = {} + + def capture_request(request): + captured_request["body"] = request.content + return httpx.Response( + 200, + json={ + "id": "chatcmpl-789", + "object": "chat.completion", + "created": 1677652288, + "model": DEFAULT_MODEL, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "A cat sitting on a couch.", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 100, "completion_tokens": 8, "total_tokens": 108}, + }, + ) + + respx_mock.post(f"{api_base}/chat/completions").mock(side_effect=capture_request) + + response = completion( + model=f"modelscope/{DEFAULT_MODEL}", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/cat.jpg"}, + }, + ], + } + ], + api_key=api_key, + api_base=api_base, + ) + + assert response.choices[0].message.content == "A cat sitting on a couch." + + body = json.loads(captured_request["body"]) + msg = body["messages"][0] + # Multimodal content should remain as a list + assert isinstance(msg["content"], list) + assert len(msg["content"]) == 2 + assert msg["content"][0] == {"type": "text", "text": "What is in this image?"} + assert msg["content"][1]["type"] == "image_url" + assert msg["content"][1]["image_url"]["url"] == "https://example.com/cat.jpg" diff --git a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py index 69d50237f72c..7f00f53c4516 100644 --- a/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py +++ b/tests/test_litellm/llms/modelscope/image_generation/test_modelscope_image_gen_transformation.py @@ -415,7 +415,6 @@ def test_get_error_class_bad_request(self): error_message="Bad request", status_code=400, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, BadRequestError) @@ -428,7 +427,6 @@ def test_get_error_class_authentication_error(self): error_message="Invalid API key", status_code=401, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, AuthenticationError) @@ -441,7 +439,6 @@ def test_get_error_class_internal_server_error(self): error_message="Internal server error", status_code=500, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, InternalServerError) @@ -454,7 +451,6 @@ def test_get_error_class_default(self): error_message="Some error", status_code=404, headers={"Content-Type": "application/json"}, - model=self.model, ) assert isinstance(error, BadRequestError) From 4b357508a774cc2bf7fd69404a301a4bf5245df6 Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Fri, 22 May 2026 10:35:29 +0800 Subject: [PATCH 6/7] fix: address PR review feedback for modelscope provider --- litellm/__init__.py | 2 +- litellm/llms/modelscope/chat/transformation.py | 11 ++++------- provider_endpoints_support.json | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index e0fb010c1d46..8ae989d4e147 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -63,6 +63,7 @@ replicate_models, clarifai_models, huggingface_models, + modelscope_models, empower_models, together_ai_models, baseten_models, @@ -625,7 +626,6 @@ def identify(event_details): deepgram_models: Set = set() elevenlabs_models: Set = set() dashscope_models: Set = set() -modelscope_models: Set = set() moonshot_models: Set = set() publicai_models: Set = set() v0_models: Set = set() diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py index 9f26bee5f540..ddc72376f8f7 100644 --- a/litellm/llms/modelscope/chat/transformation.py +++ b/litellm/llms/modelscope/chat/transformation.py @@ -2,7 +2,7 @@ Translates from OpenAI's `/v1/chat/completions` to ModelScope's `/v1/chat/completions` """ -from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, overload +from typing import Any, Coroutine, List, Literal, Optional, Tuple, Union, cast, overload from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -43,14 +43,13 @@ def _transform_messages( Messages with non-text content (e.g. image_url for vision models) are kept as lists so the parent class can normalize them properly. """ + messages = [cast(AllMessageValues, {**m}) for m in messages] for message in messages: if _has_non_text_content(message): continue content = message.get("content") if isinstance(content, list): - message["content"] = "".join( - item.get("text") or "" for item in content - ) + message["content"] = "".join(item.get("text") or "" for item in content) if is_async: return super()._transform_messages( @@ -65,9 +64,7 @@ def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: api_base = ( - api_base - or get_secret_str("MODELSCOPE_API_BASE") - or self.DEFAULT_BASE_URL + api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL ) # type: ignore dynamic_api_key = api_key or get_secret_str("MODELSCOPE_API_KEY") return api_base, dynamic_api_key diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index b4f782f9c3e2..517dd589302b 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1468,6 +1468,24 @@ "interactions": true } }, + "modelscope": { + "display_name": "ModelScope (`modelscope`)", + "url": "https://docs.litellm.ai/docs/providers/modelscope", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "moonshot": { "display_name": "Moonshot (`moonshot`)", "url": "https://docs.litellm.ai/docs/providers/moonshot", From 1e966c79bb69d747b8bfca3855e7cf53ce43ff5f Mon Sep 17 00:00:00 2001 From: yrk <2493404415@qq.com> Date: Mon, 25 May 2026 10:36:48 +0800 Subject: [PATCH 7/7] update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9924aeb58291..8bddc3190144 100644 --- a/README.md +++ b/README.md @@ -327,6 +327,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ | [Maritalk (`maritalk`)](https://docs.litellm.ai/docs/providers/maritalk) | ✅ | ✅ | ✅ | | | | | | | | | [Meta - Llama API (`meta_llama`)](https://docs.litellm.ai/docs/providers/meta_llama) | ✅ | ✅ | ✅ | | | | | | | | | [Mistral AI API (`mistral`)](https://docs.litellm.ai/docs/providers/mistral) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [ModelScope (`modelscope`)](https://docs.litellm.ai/docs/providers/modelscope) | ✅ | ✅ | ✅ | | ✅ | | | | | | | [Moonshot (`moonshot`)](https://docs.litellm.ai/docs/providers/moonshot) | ✅ | ✅ | ✅ | | | | | | | | | [Morph (`morph`)](https://docs.litellm.ai/docs/providers/morph) | ✅ | ✅ | ✅ | | | | | | | | | [Nebius AI Studio (`nebius`)](https://docs.litellm.ai/docs/providers/nebius) | ✅ | ✅ | ✅ | ✅ | | | | | | |