Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -1060,6 +1060,9 @@
"Qwen/Qwen3-235B-A22B-Thinking-2507",
# moonshotai
"moonshotai/Kimi-K2-Instruct",
"moonshotai/Kimi-K2.5",
# MiniMaxAI
"MiniMaxAI/MiniMax-M2.5",
# meta models
"meta-llama/Llama-3.1-8B-Instruct",
"meta-llama/Llama-3.3-70B-Instruct",
Expand Down
28 changes: 28 additions & 0 deletions litellm/model_prices_and_context_window_backup.json
Original file line number Diff line number Diff line change
Expand Up @@ -32399,6 +32399,34 @@
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2.5": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3e-06,
"litellm_provider": "wandb",
"mode": "chat",
"source": "https://wandb.ai/inference/coreweave/cw_moonshotai_Kimi-K2.5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"litellm_provider": "wandb",
"mode": "chat",
"source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
Expand Down
23 changes: 6 additions & 17 deletions litellm/proxy/common_utils/reset_budget_job.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import asyncio
import json
import time
from datetime import datetime, timedelta, timezone
from datetime import datetime, timezone
from typing import List, Literal, Optional, Union

from litellm._logging import verbose_proxy_logger
Expand Down Expand Up @@ -652,24 +652,13 @@ async def _reset_budget_reset_at_date(
) -> LiteLLM_BudgetTableFull:
try:
if budget.budget_duration is not None:
from litellm.litellm_core_utils.duration_parser import (
duration_in_seconds,
from litellm.proxy.common_utils.timezone_utils import (
get_budget_reset_time,
)

duration_s = duration_in_seconds(duration=budget.budget_duration)

# Fallback for existing budgets that do not have a budget_reset_at date set, ensuring the duration is taken into account
if (
budget.budget_reset_at is None
and budget.created_at + timedelta(seconds=duration_s) > current_time
):
budget.budget_reset_at = budget.created_at + timedelta(
seconds=duration_s
)
else:
budget.budget_reset_at = current_time + timedelta(
seconds=duration_s
)
budget.budget_reset_at = get_budget_reset_time(
budget_duration=budget.budget_duration
)
except Exception as e:
verbose_proxy_logger.exception(
"Error resetting budget_reset_at for budget: %s. Item: %s", e, budget
Expand Down
11 changes: 10 additions & 1 deletion litellm/proxy/hooks/model_max_budget_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,16 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
return

response_cost: float = standard_logging_payload.get("response_cost", 0)
model = standard_logging_payload.get("model")
# Use model_group (the user-facing model alias, e.g. "gpt-4o") when
# available. The enforcement path (is_key_within_model_budget) receives
# the model name from request_data["model"] which is the model group
# alias, so the spend tracking cache key must use the same name.
# Falling back to the deployment-level "model" field preserves
# behaviour for non-proxy or non-router deployments where model_group
# is None.
model = standard_logging_payload.get(
"model_group"
) or standard_logging_payload.get("model")
virtual_key = standard_logging_payload.get("metadata", {}).get(
"user_api_key_hash"
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,9 @@
"""

#### BUDGET TABLE MANAGEMENT ####
from datetime import timedelta

from fastapi import APIRouter, Depends, HTTPException

from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy._types import *
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.utils import jsonify_object
Expand Down Expand Up @@ -86,8 +84,8 @@ async def new_budget(

# if no budget_reset_at date is set, but a budget_duration is given, then set budget_reset_at initially to the first completed duration interval in future
if budget_obj.budget_reset_at is None and budget_obj.budget_duration is not None:
budget_obj.budget_reset_at = datetime.utcnow() + timedelta(
seconds=duration_in_seconds(duration=budget_obj.budget_duration)
budget_obj.budget_reset_at = get_budget_reset_time(
budget_duration=budget_obj.budget_duration
)

budget_obj_json = budget_obj.model_dump(exclude_none=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1849,7 +1849,7 @@ async def _process_single_key_update(

# Delete cache
await _delete_cache_key_object(
hashed_token=hash_token(key_update_item.key),
hashed_token=_hash_token_if_needed(key_update_item.key),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
Expand Down Expand Up @@ -3726,7 +3726,7 @@ async def _execute_virtual_key_regeneration(

if hashed_api_key or key:
await _delete_cache_key_object(
hashed_token=hash_token(key),
hashed_token=_hash_token_if_needed(key),
user_api_key_cache=user_api_key_cache,
proxy_logging_obj=proxy_logging_obj,
)
Expand Down
28 changes: 28 additions & 0 deletions model_prices_and_context_window.json
Original file line number Diff line number Diff line change
Expand Up @@ -32384,6 +32384,34 @@
"litellm_provider": "wandb",
"mode": "chat"
},
"wandb/moonshotai/Kimi-K2.5": {
"max_tokens": 262144,
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"cache_read_input_token_cost": 1e-07,
"input_cost_per_token": 6e-07,
"output_cost_per_token": 3e-06,
"litellm_provider": "wandb",
"mode": "chat",
"source": "https://wandb.ai/inference/coreweave/cw_moonshotai_Kimi-K2.5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true
},
"wandb/MiniMaxAI/MiniMax-M2.5": {
"max_tokens": 197000,
"max_input_tokens": 197000,
"max_output_tokens": 197000,
"input_cost_per_token": 3e-07,
"output_cost_per_token": 1.2e-06,
"litellm_provider": "wandb",
"mode": "chat",
"source": "https://wandb.ai/inference/coreweave/cw_MiniMaxAI_MiniMax-M2.5",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true
},
"wandb/meta-llama/Llama-3.1-8B-Instruct": {
"max_tokens": 128000,
"max_input_tokens": 128000,
Expand Down
149 changes: 149 additions & 0 deletions tests/proxy_unit_tests/test_unit_test_max_model_budget_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,155 @@ async def test_get_end_user_spend_for_model(budget_limiter):
assert spend == 50.0


@pytest.mark.asyncio
async def test_async_log_success_event_uses_model_group_for_cache_key(budget_limiter):
"""
When model_group is present in StandardLoggingPayload (proxy/router
deployments), spend must be tracked under the model_group name — not the
deployment-level model name — so the cache key matches the one used by
is_key_within_model_budget (which receives request_data["model"], the
model group alias).

Without this, providers that decorate model names (e.g. Vertex AI
"vertex_ai/claude-opus-4-6@default") track spend under a different cache
key than enforcement reads, silently disabling budget limits.
"""
from litellm.proxy.hooks.model_max_budget_limiter import (
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX,
)

virtual_key = "test-key-hash"
model_group = "claude-opus-4-6"
deployment_model = "vertex_ai/claude-opus-4-6@default"
budget_duration = "1d"
user_api_key_model_max_budget = {
model_group: {"budget_limit": 50.0, "time_period": budget_duration},
}
kwargs = {
"standard_logging_object": {
"response_cost": 0.10,
"model": deployment_model,
"model_group": model_group,
"metadata": {"user_api_key_hash": virtual_key},
},
"litellm_params": {
"metadata": {
"user_api_key_model_max_budget": user_api_key_model_max_budget,
},
},
}
with patch.object(
budget_limiter,
"_increment_spend_for_key",
new_callable=AsyncMock,
) as mock_increment:
await budget_limiter.async_log_success_event(
kwargs, response_obj=None, start_time=None, end_time=None
)
mock_increment.assert_awaited_once()
call_kwargs = mock_increment.call_args.kwargs
spend_key = call_kwargs["spend_key"]
# The cache key must use the model_group name, NOT the deployment name
assert spend_key == (
f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model_group}:{budget_duration}"
)
assert call_kwargs["response_cost"] == 0.10


@pytest.mark.asyncio
async def test_async_log_success_event_falls_back_to_model_when_no_model_group(
budget_limiter,
):
"""
When model_group is None (non-proxy / non-router usage), spend tracking
must fall back to using the model field so existing behaviour is preserved.
"""
from litellm.proxy.hooks.model_max_budget_limiter import (
VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX,
)

virtual_key = "test-key-hash"
model = "gpt-4"
budget_duration = "1d"
user_api_key_model_max_budget = {
model: {"budget_limit": 100.0, "time_period": budget_duration},
}
kwargs = {
"standard_logging_object": {
"response_cost": 0.05,
"model": model,
"model_group": None,
"metadata": {"user_api_key_hash": virtual_key},
},
"litellm_params": {
"metadata": {
"user_api_key_model_max_budget": user_api_key_model_max_budget,
},
},
}
with patch.object(
budget_limiter,
"_increment_spend_for_key",
new_callable=AsyncMock,
) as mock_increment:
await budget_limiter.async_log_success_event(
kwargs, response_obj=None, start_time=None, end_time=None
)
mock_increment.assert_awaited_once()
call_kwargs = mock_increment.call_args.kwargs
spend_key = call_kwargs["spend_key"]
assert spend_key == (
f"{VIRTUAL_KEY_SPEND_CACHE_KEY_PREFIX}:{virtual_key}:{model}:{budget_duration}"
)


@pytest.mark.asyncio
async def test_async_log_success_event_end_user_uses_model_group(budget_limiter):
"""
End-user model budget tracking must also use model_group when available,
matching the enforcement path in is_end_user_within_model_budget.
"""
from litellm.proxy.hooks.model_max_budget_limiter import (
END_USER_SPEND_CACHE_KEY_PREFIX,
)

end_user_id = "test-user"
model_group = "claude-sonnet-4-6"
deployment_model = "vertex_ai/claude-sonnet-4-6@default"
budget_duration = "1d"
user_api_key_end_user_model_max_budget = {
model_group: {"budget_limit": 25.0, "time_period": budget_duration},
}
kwargs = {
"standard_logging_object": {
"response_cost": 0.03,
"model": deployment_model,
"model_group": model_group,
"end_user": end_user_id,
"metadata": {"user_api_key_end_user_id": end_user_id},
},
"litellm_params": {
"metadata": {
"user_api_key_end_user_model_max_budget": user_api_key_end_user_model_max_budget,
},
},
}
with patch.object(
budget_limiter,
"_increment_spend_for_key",
new_callable=AsyncMock,
) as mock_increment:
await budget_limiter.async_log_success_event(
kwargs, response_obj=None, start_time=None, end_time=None
)
mock_increment.assert_awaited_once()
call_kwargs = mock_increment.call_args.kwargs
spend_key = call_kwargs["spend_key"]
assert spend_key == (
f"{END_USER_SPEND_CACHE_KEY_PREFIX}:{end_user_id}:{model_group}:{budget_duration}"
)


@pytest.mark.asyncio
async def test_async_log_success_event_uses_end_user_model_budget_duration(
budget_limiter,
Expand Down
Loading
Loading