feat(provider): add Alibaba Cloud Token Plan provider

.github/workflows/test-unit.yml

@@ -227,6 +227,7 @@ jobs:
               tests/unit/llms/a2a
               tests/unit/llms/aiml
               tests/unit/llms/aiohttp_openai
+              tests/unit/llms/alibaba_token_plan
               tests/unit/llms/apiserpent
               tests/unit/llms/aws_polly
               tests/unit/llms/azure

README.md

@@ -320,6 +320,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [AI21 (`ai21`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [AI21 Chat (`ai21_chat`)](https://docs.litellm.ai/docs/providers/ai21) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Aleph Alpha](https://docs.litellm.ai/docs/providers/aleph_alpha) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
+| [Alibaba Cloud Token Plan (`alibaba_token_plan`)](https://docs.litellm.ai/docs/providers/alibaba_token_plan) | ✅ | ✅ | ✅ |  | ✅ | ✅ | ✅ |  |  |  |
 | [Amazon Nova](https://docs.litellm.ai/docs/providers/amazon_nova) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Anthropic (`anthropic`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |
 | [Anthropic Text (`anthropic_text`)](https://docs.litellm.ai/docs/providers/anthropic) | ✅ | ✅ | ✅ |  |  |  |  |  | ✅ |  |

litellm/__init__.py

@@ -687,6 +687,7 @@ def identify(event_details):
 deepgram_models: Set = set()
 elevenlabs_models: Set = set()
 dashscope_models: Set = set()
+alibaba_token_plan_models: set[str] = set()
 qwencloud_models: Set = set()
 qwen_ai_platform_models: Set = set()
 moonshot_models: Set = set()
@@ -946,6 +947,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
             heroku_models.add(key)
         elif value.get("litellm_provider") == "dashscope":
             dashscope_models.add(key)
+        elif value.get("litellm_provider") == "alibaba_token_plan":
+            alibaba_token_plan_models.add(key)
         elif value.get("litellm_provider") == "qwencloud":
             qwencloud_models.add(key)
         elif value.get("litellm_provider") == "qwen_ai_platform":
@@ -1118,6 +1121,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
     | deepgram_models
     | elevenlabs_models
     | dashscope_models
+    | alibaba_token_plan_models
     | qwencloud_models
     | qwen_ai_platform_models
     | moonshot_models
@@ -1229,6 +1233,7 @@ def _build_models_by_provider() -> dict:
         "elevenlabs": elevenlabs_models,
         "heroku": heroku_models,
         "dashscope": dashscope_models,
+        "alibaba_token_plan": alibaba_token_plan_models,
         "qwencloud": qwencloud_models,
         "qwen_ai_platform": qwen_ai_platform_models,
         "modelscope": modelscope_models,
@@ -2146,6 +2151,30 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
     from .llms.dashscope.rerank.transformation import (
         DashScopeRerankConfig as DashScopeRerankConfig,
     )
+    from .llms.alibaba_token_plan.chat.transformation import (
+        AlibabaTokenPlanChatConfig as AlibabaTokenPlanChatConfig,
+    )
+    from .llms.alibaba_token_plan.messages.transformation import (
+        AlibabaTokenPlanAnthropicMessagesConfig as AlibabaTokenPlanAnthropicMessagesConfig,
+    )
+    from .llms.alibaba_token_plan.audio_transcription.transformation import (
+        AlibabaTokenPlanAudioTranscriptionConfig as AlibabaTokenPlanAudioTranscriptionConfig,
+    )
+    from .llms.alibaba_token_plan.text_to_speech.transformation import (
+        AlibabaTokenPlanTextToSpeechConfig as AlibabaTokenPlanTextToSpeechConfig,
+    )
+    from .llms.alibaba_token_plan.image_generation.transformation import (
+        AlibabaTokenPlanImageGenerationConfig as AlibabaTokenPlanImageGenerationConfig,
+    )
+    from .llms.alibaba_token_plan.image_edit.transformation import (
+        AlibabaTokenPlanImageEditConfig as AlibabaTokenPlanImageEditConfig,
+    )
+    from .llms.alibaba_token_plan.realtime.transformation import (
+        AlibabaTokenPlanRealtimeConfig as AlibabaTokenPlanRealtimeConfig,
+    )
+    from .llms.alibaba_token_plan.videos.transformation import (
+        AlibabaTokenPlanVideoConfig as AlibabaTokenPlanVideoConfig,
+    )
     from .llms.dashscope.qwencloud import (
         QwenCloudChatConfig as QwenCloudChatConfig,
     )

litellm/_lazy_imports_registry.py

@@ -330,6 +330,14 @@
     "GigaChatConfig",
     "GigaChatEmbeddingConfig",
     "DashScopeChatConfig",
+    "AlibabaTokenPlanChatConfig",
+    "AlibabaTokenPlanAnthropicMessagesConfig",
+    "AlibabaTokenPlanAudioTranscriptionConfig",
+    "AlibabaTokenPlanTextToSpeechConfig",
+    "AlibabaTokenPlanImageGenerationConfig",
+    "AlibabaTokenPlanImageEditConfig",
+    "AlibabaTokenPlanRealtimeConfig",
+    "AlibabaTokenPlanVideoConfig",
     "QwenCloudChatConfig",
     "QwenAIPlatformChatConfig",
     "ModelScopeChatConfig",
@@ -1251,6 +1259,38 @@
         ".llms.dashscope.chat.transformation",
         "DashScopeChatConfig",
     ),
+    "AlibabaTokenPlanChatConfig": (
+        ".llms.alibaba_token_plan.chat.transformation",
+        "AlibabaTokenPlanChatConfig",
+    ),
+    "AlibabaTokenPlanAnthropicMessagesConfig": (
+        ".llms.alibaba_token_plan.messages.transformation",
+        "AlibabaTokenPlanAnthropicMessagesConfig",
+    ),
+    "AlibabaTokenPlanAudioTranscriptionConfig": (
+        ".llms.alibaba_token_plan.audio_transcription.transformation",
+        "AlibabaTokenPlanAudioTranscriptionConfig",
+    ),
+    "AlibabaTokenPlanTextToSpeechConfig": (
+        ".llms.alibaba_token_plan.text_to_speech.transformation",
+        "AlibabaTokenPlanTextToSpeechConfig",
+    ),
+    "AlibabaTokenPlanImageGenerationConfig": (
+        ".llms.alibaba_token_plan.image_generation.transformation",
+        "AlibabaTokenPlanImageGenerationConfig",
+    ),
+    "AlibabaTokenPlanImageEditConfig": (
+        ".llms.alibaba_token_plan.image_edit.transformation",
+        "AlibabaTokenPlanImageEditConfig",
+    ),
+    "AlibabaTokenPlanRealtimeConfig": (
+        ".llms.alibaba_token_plan.realtime.transformation",
+        "AlibabaTokenPlanRealtimeConfig",
+    ),
+    "AlibabaTokenPlanVideoConfig": (
+        ".llms.alibaba_token_plan.videos.transformation",
+        "AlibabaTokenPlanVideoConfig",
+    ),
     "QwenCloudChatConfig": (
         ".llms.dashscope.qwencloud",
         "QwenCloudChatConfig",

litellm/constants.py

@@ -805,6 +805,7 @@ class ServerStreamingClassification(str, Enum):
     "nscale",
     "nebius",
     "dashscope",
+    "alibaba_token_plan",
     "qwencloud",
     "qwen_ai_platform",
     "modelscope",
@@ -1057,6 +1058,7 @@ class ServerStreamingClassification(str, Enum):
     "nscale",
     "nebius",
     "dashscope",
+    "alibaba_token_plan",
     "qwencloud",
     "qwen_ai_platform",
     "modelscope",

litellm/images/main.py

@@ -381,6 +381,7 @@ def image_generation(
             litellm.LlmProviders.VERTEX_AI,
             litellm.LlmProviders.OPENROUTER,
             litellm.LlmProviders.DASHSCOPE,
+            litellm.LlmProviders.ALIBABA_TOKEN_PLAN,
             litellm.LlmProviders.QWENCLOUD,
             litellm.LlmProviders.QWEN_AI_PLATFORM,
             litellm.LlmProviders.EDENAI,

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -600,7 +600,7 @@ def _get_openai_compatible_provider_info(
     from litellm.llms.openai_like.dynamic_config import create_config_class
     from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 
-    if JSONProviderRegistry.exists(custom_llm_provider):
+    if custom_llm_provider != "alibaba_token_plan" and JSONProviderRegistry.exists(custom_llm_provider):
         provider_config: Final = JSONProviderRegistry.get(custom_llm_provider)
         if provider_config is None:
             raise ValueError(f"Provider {custom_llm_provider} not found")
@@ -859,6 +859,10 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.HerokuChatConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "alibaba_token_plan":
+        from litellm.llms.alibaba_token_plan.common_utils import get_api_base, get_api_key
+
+        return model, custom_llm_provider, get_api_key(api_key), get_api_base(api_base)
     elif custom_llm_provider in ("dashscope", "qwencloud", "qwen_ai_platform"):
         (
             api_base,

litellm/llms/alibaba_token_plan/audio_transcription/transformation.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import base64
+from collections.abc import Mapping, Sequence
+from pathlib import PurePath
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, ValidationError
+
+import litellm
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.alibaba_token_plan.common_utils import IMAGE_PATH, get_api_url, validate_headers
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai import AllMessageValues, OpenAIAudioTranscriptionOptionalParams
+from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageDurationObject
+
+SUPPORTED_RESPONSE_FORMATS: Final = ("json", "text")
+
+
+class _TranscriptionOutput(BaseModel, frozen=True):
+    text: str
+
+
+class _TranscriptionUsage(BaseModel, frozen=True):
+    duration: float
+
+
+class _TranscriptionResult(BaseModel, frozen=True):
+    output: _TranscriptionOutput
+    usage: _TranscriptionUsage | None = None
+
+
+class AlibabaTokenPlanAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+    @property
+    def has_native_transcription_endpoint(self) -> bool:
+        return True
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAIAudioTranscriptionOptionalParams]:  # mutable-ok: base config requires a list
+        return ["language", "prompt", "response_format"]
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: the base config contract returns mutable request parameters
+        response_format: Final = non_default_params.get("response_format")
+        unsupported: Final = response_format is not None and response_format not in SUPPORTED_RESPONSE_FORMATS
+        if unsupported and not (drop_params or litellm.drop_params):
+            raise UnsupportedParamsError(
+                status_code=400,
+                message=(
+                    f"Alibaba Token Plan transcription does not support response_format={response_format!r}. "
+                    f"Supported values: {', '.join(SUPPORTED_RESPONSE_FORMATS)}. "
+                    "To drop unsupported openai params from the call, set `litellm.drop_params = True`"
+                ),
+            )
+        return {**optional_params, **{k: v for k, v in non_default_params.items() if k in ("language", "prompt")}}
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: the HTTP handler adds headers after validation
+        return validate_headers(headers, api_key)
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return get_api_url(api_base, IMAGE_PATH)
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
+    ) -> BaseLLMException:
+        return BaseLLMException(message=error_message, status_code=status_code, headers=dict(headers))
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> AudioTranscriptionRequestData:
+        audio: Final = process_audio_file(audio_file)
+        audio_data: Final = f"data:{audio.content_type};base64,{base64.b64encode(audio.file_content).decode('ascii')}"
+        prompt: Final = optional_params.get("prompt")
+        language: Final = optional_params.get("language")
+        context: Final = [{"role": "user", "content": [{"type": "input_text", "text": prompt}]}] if prompt else []
+        return AudioTranscriptionRequestData(
+            data={
+                "model": model,
+                "input": {
+                    "messages": [
+                        *context,
+                        {"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": audio_data}}]},
+                    ]
+                },
+                "parameters": {
+                    "format": PurePath(audio.filename).suffix.lstrip(".").lower() or "wav",
+                    **({"language_hints": [language]} if language else {}),
+                },
+            },
+            content_type="application/json",
+        )
+
+    def transform_audio_transcription_response(self, raw_response: httpx.Response) -> TranscriptionResponse:
+        if not raw_response.is_success:
+            raise self.get_error_class(raw_response.text, raw_response.status_code, raw_response.headers)
+        try:
+            parsed: Final = _TranscriptionResult.model_validate_json(raw_response.content)
+        except ValidationError:
+            raise self.get_error_class(raw_response.text, 502, raw_response.headers) from None
+        response: Final = TranscriptionResponse(text=parsed.output.text)
+        if parsed.usage is not None:
+            response.usage = TranscriptionUsageDurationObject(type="duration", seconds=parsed.usage.duration)
+            response["duration"] = parsed.usage.duration
+        return response

litellm/llms/alibaba_token_plan/chat/transformation.py

@@ -0,0 +1,12 @@
+from litellm.llms.alibaba_token_plan.common_utils import get_api_base, get_api_key
+from litellm.llms.dashscope.chat.transformation import DashScopeChatConfig
+
+
+class AlibabaTokenPlanChatConfig(DashScopeChatConfig):
+    def _get_openai_compatible_provider_info(
+        self, api_base: str | None, api_key: str | None
+    ) -> tuple[str | None, str | None]:
+        return self._resolve_chat_api_base(api_base), get_api_key(api_key)
+
+    def _resolve_chat_api_base(self, api_base: str | None) -> str:
+        return get_api_base(api_base)