diff --git a/.circleci/config.yml b/.circleci/config.yml index 38e6d1fc332..12e3cb1f6b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3218,6 +3218,7 @@ jobs: -e DD_API_KEY=$DD_API_KEY \ -e DD_SITE=$DD_SITE \ -e LITELLM_LICENSE=$LITELLM_LICENSE \ + -e LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/litellm/proxy/example_config_yaml/pass_through_config.yaml:/app/config.yaml \ diff --git a/CLAUDE.md b/CLAUDE.md index 5395d6d938e..d9061b5e2be 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,4 +156,4 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components: **Fix options:** 1. **Create a Prisma migration** (permanent) — run `prisma migrate dev --name ` in the worktree. The generated file will be picked up by `prisma migrate deploy` on next startup. 2. **Apply manually for local dev** — `psql -d litellm -c "ALTER TABLE ... ADD COLUMN IF NOT EXISTS ..."` after each proxy start. Fine for dev, not for production. -3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. \ No newline at end of file +3. **Update litellm-proxy-extras** — if the package is installed from PyPI, its migration directory must include the new file. Either update the package or run the migration manually until the next release ships it. diff --git a/ci_cd/security_scans.sh b/ci_cd/security_scans.sh index 62440d13ebb..e0f370e0035 100755 --- a/ci_cd/security_scans.sh +++ b/ci_cd/security_scans.sh @@ -11,7 +11,7 @@ echo "Starting security scans for LiteLLM..." install_trivy() { echo "Installing Trivy and required tools..." sudo apt-get update - sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl + sudo apt-get install -y wget apt-transport-https gnupg lsb-release jq curl bsdmainutils wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add - echo "deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/trivy.list sudo apt-get update diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index a3ef7b264ec..e0e1e35b94e 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -660,7 +660,14 @@ def _select_model_name_for_cost_calc( if custom_pricing is True: if router_model_id is not None and router_model_id in litellm.model_cost: - return_model = router_model_id + entry = litellm.model_cost[router_model_id] + if ( + entry.get("input_cost_per_token") is not None + or entry.get("input_cost_per_second") is not None + ): + return_model = router_model_id + else: + return_model = model else: return_model = model diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404/index.html similarity index 100% rename from litellm/proxy/_experimental/out/404.html rename to litellm/proxy/_experimental/out/404/index.html diff --git a/litellm/proxy/_experimental/out/_not-found.html b/litellm/proxy/_experimental/out/_not-found/index.html similarity index 100% rename from litellm/proxy/_experimental/out/_not-found.html rename to litellm/proxy/_experimental/out/_not-found/index.html diff --git a/litellm/proxy/_experimental/out/api-reference.html b/litellm/proxy/_experimental/out/api-reference/index.html similarity index 100% rename from litellm/proxy/_experimental/out/api-reference.html rename to litellm/proxy/_experimental/out/api-reference/index.html diff --git a/litellm/proxy/_experimental/out/experimental/api-playground.html b/litellm/proxy/_experimental/out/experimental/api-playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/api-playground.html rename to litellm/proxy/_experimental/out/experimental/api-playground/index.html diff --git a/litellm/proxy/_experimental/out/experimental/budgets.html b/litellm/proxy/_experimental/out/experimental/budgets/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/budgets.html rename to litellm/proxy/_experimental/out/experimental/budgets/index.html diff --git a/litellm/proxy/_experimental/out/experimental/caching.html b/litellm/proxy/_experimental/out/experimental/caching/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/caching.html rename to litellm/proxy/_experimental/out/experimental/caching/index.html diff --git a/litellm/proxy/_experimental/out/experimental/claude-code-plugins.html b/litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/claude-code-plugins.html rename to litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html diff --git a/litellm/proxy/_experimental/out/experimental/old-usage.html b/litellm/proxy/_experimental/out/experimental/old-usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/old-usage.html rename to litellm/proxy/_experimental/out/experimental/old-usage/index.html diff --git a/litellm/proxy/_experimental/out/experimental/prompts.html b/litellm/proxy/_experimental/out/experimental/prompts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/prompts.html rename to litellm/proxy/_experimental/out/experimental/prompts/index.html diff --git a/litellm/proxy/_experimental/out/experimental/tag-management.html b/litellm/proxy/_experimental/out/experimental/tag-management/index.html similarity index 100% rename from litellm/proxy/_experimental/out/experimental/tag-management.html rename to litellm/proxy/_experimental/out/experimental/tag-management/index.html diff --git a/litellm/proxy/_experimental/out/guardrails.html b/litellm/proxy/_experimental/out/guardrails/index.html similarity index 100% rename from litellm/proxy/_experimental/out/guardrails.html rename to litellm/proxy/_experimental/out/guardrails/index.html diff --git a/litellm/proxy/_experimental/out/login.html b/litellm/proxy/_experimental/out/login/index.html similarity index 100% rename from litellm/proxy/_experimental/out/login.html rename to litellm/proxy/_experimental/out/login/index.html diff --git a/litellm/proxy/_experimental/out/logs.html b/litellm/proxy/_experimental/out/logs/index.html similarity index 100% rename from litellm/proxy/_experimental/out/logs.html rename to litellm/proxy/_experimental/out/logs/index.html diff --git a/litellm/proxy/_experimental/out/mcp/oauth/callback.html b/litellm/proxy/_experimental/out/mcp/oauth/callback/index.html similarity index 100% rename from litellm/proxy/_experimental/out/mcp/oauth/callback.html rename to litellm/proxy/_experimental/out/mcp/oauth/callback/index.html diff --git a/litellm/proxy/_experimental/out/model-hub.html b/litellm/proxy/_experimental/out/model-hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model-hub.html rename to litellm/proxy/_experimental/out/model-hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub.html b/litellm/proxy/_experimental/out/model_hub/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub.html rename to litellm/proxy/_experimental/out/model_hub/index.html diff --git a/litellm/proxy/_experimental/out/model_hub_table.html b/litellm/proxy/_experimental/out/model_hub_table/index.html similarity index 100% rename from litellm/proxy/_experimental/out/model_hub_table.html rename to litellm/proxy/_experimental/out/model_hub_table/index.html diff --git a/litellm/proxy/_experimental/out/models-and-endpoints.html b/litellm/proxy/_experimental/out/models-and-endpoints/index.html similarity index 100% rename from litellm/proxy/_experimental/out/models-and-endpoints.html rename to litellm/proxy/_experimental/out/models-and-endpoints/index.html diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding/index.html similarity index 100% rename from litellm/proxy/_experimental/out/onboarding.html rename to litellm/proxy/_experimental/out/onboarding/index.html diff --git a/litellm/proxy/_experimental/out/organizations.html b/litellm/proxy/_experimental/out/organizations/index.html similarity index 100% rename from litellm/proxy/_experimental/out/organizations.html rename to litellm/proxy/_experimental/out/organizations/index.html diff --git a/litellm/proxy/_experimental/out/playground.html b/litellm/proxy/_experimental/out/playground/index.html similarity index 100% rename from litellm/proxy/_experimental/out/playground.html rename to litellm/proxy/_experimental/out/playground/index.html diff --git a/litellm/proxy/_experimental/out/policies.html b/litellm/proxy/_experimental/out/policies/index.html similarity index 100% rename from litellm/proxy/_experimental/out/policies.html rename to litellm/proxy/_experimental/out/policies/index.html diff --git a/litellm/proxy/_experimental/out/settings/admin-settings.html b/litellm/proxy/_experimental/out/settings/admin-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/admin-settings.html rename to litellm/proxy/_experimental/out/settings/admin-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/logging-and-alerts.html b/litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/logging-and-alerts.html rename to litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html diff --git a/litellm/proxy/_experimental/out/settings/router-settings.html b/litellm/proxy/_experimental/out/settings/router-settings/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/router-settings.html rename to litellm/proxy/_experimental/out/settings/router-settings/index.html diff --git a/litellm/proxy/_experimental/out/settings/ui-theme.html b/litellm/proxy/_experimental/out/settings/ui-theme/index.html similarity index 100% rename from litellm/proxy/_experimental/out/settings/ui-theme.html rename to litellm/proxy/_experimental/out/settings/ui-theme/index.html diff --git a/litellm/proxy/_experimental/out/teams.html b/litellm/proxy/_experimental/out/teams/index.html similarity index 100% rename from litellm/proxy/_experimental/out/teams.html rename to litellm/proxy/_experimental/out/teams/index.html diff --git a/litellm/proxy/_experimental/out/test-key.html b/litellm/proxy/_experimental/out/test-key/index.html similarity index 100% rename from litellm/proxy/_experimental/out/test-key.html rename to litellm/proxy/_experimental/out/test-key/index.html diff --git a/litellm/proxy/_experimental/out/tools/mcp-servers.html b/litellm/proxy/_experimental/out/tools/mcp-servers/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/mcp-servers.html rename to litellm/proxy/_experimental/out/tools/mcp-servers/index.html diff --git a/litellm/proxy/_experimental/out/tools/vector-stores.html b/litellm/proxy/_experimental/out/tools/vector-stores/index.html similarity index 100% rename from litellm/proxy/_experimental/out/tools/vector-stores.html rename to litellm/proxy/_experimental/out/tools/vector-stores/index.html diff --git a/litellm/proxy/_experimental/out/usage.html b/litellm/proxy/_experimental/out/usage/index.html similarity index 100% rename from litellm/proxy/_experimental/out/usage.html rename to litellm/proxy/_experimental/out/usage/index.html diff --git a/litellm/proxy/_experimental/out/users.html b/litellm/proxy/_experimental/out/users/index.html similarity index 100% rename from litellm/proxy/_experimental/out/users.html rename to litellm/proxy/_experimental/out/users/index.html diff --git a/litellm/proxy/_experimental/out/virtual-keys.html b/litellm/proxy/_experimental/out/virtual-keys/index.html similarity index 100% rename from litellm/proxy/_experimental/out/virtual-keys.html rename to litellm/proxy/_experimental/out/virtual-keys/index.html diff --git a/litellm/proxy/example_config_yaml/custom_auth_basic.py b/litellm/proxy/example_config_yaml/custom_auth_basic.py index 4d633a54fe2..0da6105a305 100644 --- a/litellm/proxy/example_config_yaml/custom_auth_basic.py +++ b/litellm/proxy/example_config_yaml/custom_auth_basic.py @@ -1,6 +1,6 @@ from fastapi import Request -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: @@ -9,6 +9,7 @@ async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: api_key="best-api-key-ever", user_id="best-user-id-ever", team_id="best-team-id-ever", + user_role=LitellmUserRoles.PROXY_ADMIN, ) except Exception: raise Exception diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 701762c834c..e5a34ae8bdd 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -340,9 +340,16 @@ def _maybe_setup_prometheus_multiproc_dir( return # Check if prometheus is in any callback list + # Each setting can be a list or a single string; normalize to list callbacks = litellm_settings.get("callbacks") or [] success_callbacks = litellm_settings.get("success_callback") or [] failure_callbacks = litellm_settings.get("failure_callback") or [] + if isinstance(callbacks, str): + callbacks = [callbacks] + if isinstance(success_callbacks, str): + success_callbacks = [success_callbacks] + if isinstance(failure_callbacks, str): + failure_callbacks = [failure_callbacks] all_callbacks = callbacks + success_callbacks + failure_callbacks if "prometheus" not in all_callbacks: return diff --git a/pyproject.toml b/pyproject.toml index 8efc6366128..07004f3ae16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.82.2" +version = "1.82.3" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -183,7 +183,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.82.2" +version = "1.82.3" version_files = [ "pyproject.toml:^version" ] diff --git a/tests/local_testing/test_router_utils.py b/tests/local_testing/test_router_utils.py index 9d51685751a..4f7f53cef02 100644 --- a/tests/local_testing/test_router_utils.py +++ b/tests/local_testing/test_router_utils.py @@ -199,6 +199,7 @@ def test_router_get_model_info_wildcard_routes(): @pytest.mark.asyncio +@pytest.mark.flaky(retries=3, delay=1) async def test_router_get_model_group_usage_wildcard_routes(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") @@ -219,7 +220,7 @@ async def test_router_get_model_group_usage_wildcard_routes(): ) print(resp) - await asyncio.sleep(1) + await asyncio.sleep(2) tpm, rpm = await router.get_model_group_usage(model_group="gemini/gemini-1.5-flash") diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 2544e06598e..a4a28215e16 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -395,6 +395,7 @@ async def test_mcp_http_transport_tool_not_found(): @pytest.mark.asyncio async def test_streamable_http_mcp_handler_mock(): """Test the streamable HTTP MCP handler functionality""" + from litellm.proxy._types import UserAPIKeyAuth # Mock the session manager and its methods mock_session_manager = AsyncMock() @@ -425,6 +426,8 @@ async def test_streamable_http_mcp_handler_mock(): ), patch( "litellm.proxy._experimental.mcp_server.server.extract_mcp_auth_context", AsyncMock(return_value=mock_auth_context), + ), patch( + "litellm.proxy._experimental.mcp_server.server.set_auth_context", ): from litellm.proxy._experimental.mcp_server.server import ( handle_streamable_http_mcp, diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py index b3d785f1133..0a67d5e64e0 100644 --- a/tests/test_litellm/proxy/test_prometheus_cleanup.py +++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py @@ -67,6 +67,30 @@ def test_respects_existing_env_var(self, tmp_path): assert os.environ["PROMETHEUS_MULTIPROC_DIR"] == custom_dir assert os.path.isdir(custom_dir) + @pytest.mark.parametrize( + "litellm_settings", + [ + {"callbacks": "prometheus"}, + {"success_callback": "prometheus"}, + {"failure_callback": "prometheus"}, + {"callbacks": "custom_callback"}, # string but not prometheus + ], + ) + def test_handles_string_callbacks(self, litellm_settings): + """When callbacks are specified as a string instead of a list, should not crash.""" + with patch.dict(os.environ, {}, clear=False): + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + os.environ.pop("prometheus_multiproc_dir", None) + + # Should not raise TypeError + ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir( + num_workers=4, + litellm_settings=litellm_settings, + ) + + # Cleanup + os.environ.pop("PROMETHEUS_MULTIPROC_DIR", None) + @pytest.mark.parametrize( "num_workers, litellm_settings", [ diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 642d21a42f7..c5d6c45f9a5 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -677,7 +677,7 @@ def test_startup_fails_when_db_setup_fails( mock_atexit_register, mock_subprocess_run, ): - """Test that proxy exits with code 1 when PrismaManager.setup_database returns False""" + """Test that proxy exits with code 1 when PrismaManager.setup_database returns False and --enforce_prisma_migration_check is set""" from litellm.proxy.proxy_cli import run_server mock_subprocess_run.return_value = MagicMock(returncode=0) @@ -717,7 +717,7 @@ def test_startup_fails_when_db_setup_fails( with pytest.raises(SystemExit) as exc_info: run_server.main( - ["--local", "--skip_server_startup"], standalone_mode=False + ["--local", "--skip_server_startup", "--enforce_prisma_migration_check"], standalone_mode=False ) assert exc_info.value.code == 1 mock_setup_database.assert_called_once_with(use_migrate=True) diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py index 463f6952d23..8f5c3ece0ca 100644 --- a/tests/test_litellm/test_cost_calculator.py +++ b/tests/test_litellm/test_cost_calculator.py @@ -388,6 +388,65 @@ def test_custom_pricing_cost_calc_uses_router_model_id_from_litellm_metadata(): assert custom_model_id not in (selected_model_no_custom or "") +def test_per_request_custom_pricing_with_router(): + """When custom pricing is passed as per-request kwargs (not in model_list), + _select_model_name_for_cost_calc should fall back to the model name + (where register_model stored the pricing) instead of the router_model_id + (which has no pricing data). + + Regression test for the bug where response._hidden_params["response_cost"] + returned 0.0 for per-request custom pricing via Router. + """ + from litellm import Router + from litellm.cost_calculator import _select_model_name_for_cost_calc + + router = Router( + model_list=[ + { + "model_name": "openai/gpt-3.5-turbo", + "litellm_params": { + "model": "openai/gpt-3.5-turbo", + "api_key": "test_api_key", + }, + }, + ] + ) + + # Get the deployment's model_id (hash) that the router registered + deployment = router.model_list[0] + router_model_id = deployment["model_info"]["id"] + + # The router registered this hash in model_cost but without custom pricing + assert router_model_id in litellm.model_cost + entry = litellm.model_cost[router_model_id] + # No custom pricing was set in model_list, so these should be None + assert entry.get("input_cost_per_token") is None + + # Now simulate what completion() does: register custom pricing under the model name + litellm.register_model( + { + "openai/gpt-3.5-turbo": { + "input_cost_per_token": 2.0, + "output_cost_per_token": 2.0, + "litellm_provider": "openai", + } + } + ) + + # _select_model_name_for_cost_calc should pick the model name (which has pricing), + # NOT the router_model_id (which has no pricing) + selected = _select_model_name_for_cost_calc( + model="openai/gpt-3.5-turbo", + completion_response=None, + custom_pricing=True, + custom_llm_provider="openai", + router_model_id=router_model_id, + ) + assert selected is not None + assert router_model_id not in selected + assert "gpt-3.5-turbo" in selected + + def test_azure_realtime_cost_calculator(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 400e86d19ee..4e7153b64cd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -228,7 +228,7 @@ export function useLogFilterLogic({ const filteredLogs: PaginatedResponse = useMemo(() => { if (hasBackendFilters) { // Prefer backend result if present; otherwise fall back to latest logs - if (backendFilteredLogs && backendFilteredLogs.data && backendFilteredLogs.data.length > 0) { + if (backendFilteredLogs && backendFilteredLogs.data) { return backendFilteredLogs; } return (