feat: litellm oss 110626

litellm/__init__.py

@@ -43,6 +43,7 @@ def _dev_env_hot_reload_enabled() -> bool:
     Type,
 )
 from litellm.types.integrations.datadog import DatadogInitParams
+from litellm.types.integrations.newrelic import NewRelicInitParams
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,
@@ -154,10 +155,12 @@ def _dev_env_hot_reload_enabled() -> bool:
     "gitlab",
     "cloudzero",
     "focus",
+    "mavvrik",
     "vantage",
     "posthog",
     "levo",
     "compression_interception",
+    "newrelic",
 ]
 cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
@@ -412,6 +415,7 @@ def _dev_env_hot_reload_enabled() -> bool:
 s3_audit_callback_params: Optional[Dict] = None
 datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
 datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
+newrelic_params: Optional[Union[NewRelicInitParams, Dict]] = None
 aws_sqs_callback_params: Optional[Dict] = None
 generic_logger_headers: Optional[Dict] = None
 default_key_generate_params: Optional[Dict] = None
@@ -1373,6 +1377,7 @@ def add_known_models(model_cost_map: Optional[Dict] = None):
 from .realtime_api.main import (
     _arealtime,
     acreate_realtime_client_secret,
+    acreate_realtime_transcription_session,
     arealtime_calls,
 )
 from .responses.main import _aresponses_websocket

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -171,6 +171,8 @@ def completion(self, *args, **kwargs) -> Union[
         model_response = validated_kwargs["model_response"]
         logging_obj = validated_kwargs["logging_obj"]
         custom_llm_provider = validated_kwargs["custom_llm_provider"]
+        if kwargs.get("stream") is True and "stream" not in optional_params:
+            optional_params = {**optional_params, "stream": True}
 
         request_data = self.transformation_handler.transform_request(
             model=model,
@@ -263,6 +265,8 @@ async def acompletion(
         model_response = validated_kwargs["model_response"]
         logging_obj = validated_kwargs["logging_obj"]
         custom_llm_provider = validated_kwargs["custom_llm_provider"]
+        if kwargs.get("stream") is True and "stream" not in optional_params:
+            optional_params = {**optional_params, "stream": True}
 
         try:
             request_data = self.transformation_handler.transform_request(

litellm/constants.py

@@ -421,6 +421,7 @@
 DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS = int(
     os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096)
 )
+DEFAULT_OCI_CHAT_MAX_TOKENS = 4096
 TOGETHER_AI_4_B = int(os.getenv("TOGETHER_AI_4_B", 4))
 TOGETHER_AI_8_B = int(os.getenv("TOGETHER_AI_8_B", 8))
 TOGETHER_AI_21_B = int(os.getenv("TOGETHER_AI_21_B", 21))
@@ -1483,6 +1484,7 @@
 DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME = "db_daily_tag_spend_update_job"
 PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME = "prometheus_emit_budget_metrics"
 CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME = "cloudzero_export_usage_data"
+MAVVRIK_FOCUS_EXPORT_JOB_NAME = "mavvrik_focus_export_usage_data"
 CLOUDZERO_MAX_FETCHED_DATA_RECORDS = int(
     os.getenv("CLOUDZERO_MAX_FETCHED_DATA_RECORDS", 50000)
 )

litellm/cost_calculator.py

@@ -2488,6 +2488,11 @@ def create_logging_realtime_object(
         )
 
 
+_TRANSCRIPTION_COMPLETED_EVENT_TYPE = (
+    "conversation.item.input_audio_transcription.completed"
+)
+
+
 def handle_realtime_stream_cost_calculation(
     results: OpenAIRealtimeStreamList,
     combined_usage_object: Usage,
@@ -2533,4 +2538,99 @@ def handle_realtime_stream_cost_calculation(
         break  # exit if we find a valid model
     total_cost = input_cost_per_token + output_cost_per_token
 
+    if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results):
+        total_cost += handle_realtime_transcription_cost_calculation(
+            results=results,
+            custom_llm_provider=custom_llm_provider,
+            litellm_model_name=litellm_model_name,
+        )
+
+    return total_cost
+
+
+def handle_realtime_transcription_cost_calculation(
+    results: OpenAIRealtimeStreamList,
+    custom_llm_provider: str,
+    litellm_model_name: str,
+) -> float:
+    """
+    Cost for realtime transcription sessions (e.g. gpt-realtime-whisper).
+
+    Transcription sessions emit no `response.done` events; instead each
+    `conversation.item.input_audio_transcription.completed` event carries a
+    `usage` object billed by the ASR model. The usage is one of:
+      - {"type": "duration", "seconds": <float>}  → priced via input_cost_per_second
+      - {"type": "tokens", "input_tokens": ...}    → priced via input/audio token cost
+    """
+    completed_events = [
+        cast(dict, result)
+        for result in results
+        if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
+    ]
+    if not completed_events:
+        return 0.0
+
+    model_name = (
+        _get_transcription_model_name_from_results(results) or litellm_model_name
+    )
+    try:
+        model_info = litellm.get_model_info(
+            model=model_name, custom_llm_provider=custom_llm_provider
+        )
+    except Exception:
+        model_info = None
+
+    total_cost = 0.0
+    for event in completed_events:
+        usage = event.get("usage") or {}
+        total_cost += _transcription_usage_cost(usage, model_info)
     return total_cost
+
+
+def _get_transcription_model_name_from_results(
+    results: OpenAIRealtimeStreamList,
+) -> Optional[str]:
+    """Resolve the ASR model from a transcription_session.* / session.* event."""
+    for result in results:
+        if result.get("type") in (
+            "transcription_session.created",
+            "transcription_session.updated",
+            "session.created",
+            "session.updated",
+        ):
+            session = cast(dict, result).get("session", {}) or {}
+            transcription = (
+                (session.get("audio", {}) or {}).get("input", {}) or {}
+            ).get("transcription", {}) or session.get("input_audio_transcription", {})
+            model = (transcription or {}).get("model") or session.get("model")
+            if model:
+                return model
+    return None
+
+
+def _transcription_usage_cost(usage: dict, model_info: Optional[ModelInfo]) -> float:
+    if model_info is None:
+        return 0.0
+    usage_type = usage.get("type")
+    if usage_type == "duration":
+        seconds = usage.get("seconds") or 0.0
+        per_second = model_info.get("input_cost_per_second") or 0.0
+        return float(seconds) * float(per_second)
+    if usage_type == "tokens":
+        input_token_details = usage.get("input_token_details") or {}
+        audio_tokens = input_token_details.get("audio_tokens") or 0
+        text_tokens = input_token_details.get("text_tokens") or 0
+        output_tokens = usage.get("output_tokens") or 0
+        audio_cost = float(audio_tokens) * float(
+            model_info.get("input_cost_per_audio_token")
+            or model_info.get("input_cost_per_token")
+            or 0.0
+        )
+        text_cost = float(text_tokens) * float(
+            model_info.get("input_cost_per_token") or 0.0
+        )
+        output_cost = float(output_tokens) * float(
+            model_info.get("output_cost_per_token") or 0.0
+        )
+        return audio_cost + text_cost + output_cost
+    return 0.0

litellm/integrations/SlackAlerting/budget_alert_types.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from typing import Literal
 
-from litellm.proxy._types import CallInfo
+from litellm.proxy._types import CallInfo, Litellm_EntityType
 
 
 class BaseBudgetAlertType(ABC):
@@ -31,6 +31,8 @@ def get_event_message(self) -> str:
         return "Soft Budget Crossed: "
 
     def get_id(self, user_info: CallInfo) -> str:
+        if user_info.event_group == Litellm_EntityType.TEAM:
+            return user_info.team_id or "default_id"
         return user_info.token or "default_id"
 
 

litellm/integrations/callback_configs.json

@@ -290,6 +290,21 @@
     },
     "description": "Langsmith Logging Integration"
   },
+  {
+    "id": "newrelic",
+    "displayName": "New Relic",
+    "logo": "newrelic.png",
+    "supports_key_team_logging": false,
+    "dynamic_params": {
+      "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED": {
+        "type": "text",
+        "ui_name": "Record AI Content (default: true)",
+        "description": "Whether to record AI message content. Set to false to disable.",
+        "required": false
+      }
+    },
+    "description": "New Relic AI Monitoring Integration"
+  },
   {
     "id": "openmeter",
     "displayName": "OpenMeter",

litellm/integrations/focus/destinations/__init__.py

@@ -4,6 +4,7 @@
 from .factory import FocusDestinationFactory
 from .gcs_destination import FocusGCSDestination
 from .s3_destination import FocusS3Destination
+from .mavvrik_destination import FocusMavvrikDestination
 from .vantage_destination import FocusVantageDestination
 
 __all__ = [
@@ -12,5 +13,6 @@
     "FocusGCSDestination",
     "FocusTimeWindow",
     "FocusS3Destination",
+    "FocusMavvrikDestination",
     "FocusVantageDestination",
 ]

litellm/integrations/focus/destinations/factory.py

@@ -8,6 +8,7 @@
 from .base import FocusDestination
 from .gcs_destination import FocusGCSDestination
 from .s3_destination import FocusS3Destination
+from .mavvrik_destination import FocusMavvrikDestination
 from .vantage_destination import FocusVantageDestination
 
 
@@ -32,6 +33,8 @@ def create(
             return FocusVantageDestination(prefix=prefix, config=normalized_config)
         if provider_lower == "gcs":
             return FocusGCSDestination(prefix=prefix, config=normalized_config)
+        if provider_lower == "mavvrik":
+            return FocusMavvrikDestination(prefix=prefix, config=normalized_config)
         raise NotImplementedError(
             f"Provider '{provider}' not supported for Focus export"
         )
@@ -87,6 +90,15 @@ def _resolve_config(
                     "FOCUS_GCS_BUCKET_NAME must be provided for GCS exports"
                 )
             return {k: v for k, v in resolved.items() if v is not None}
+        if provider == "mavvrik":
+            resolved = {
+                "api_key": overrides.get("api_key") or os.getenv("MAVVRIK_API_KEY"),
+                "api_endpoint": overrides.get("api_endpoint")
+                or os.getenv("MAVVRIK_API_ENDPOINT"),
+                "connection_id": overrides.get("connection_id")
+                or os.getenv("MAVVRIK_CONNECTION_ID"),
+            }
+            return {k: v for k, v in resolved.items() if v is not None}
         raise NotImplementedError(
             f"Provider '{provider}' not supported for Focus export configuration"
         )