From a8a0c837e63f76c2335e5ff08db79553ab65f2cf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 11:58:44 -0700 Subject: [PATCH 01/23] fix(common_daily_activity.py): initial commit with working mock BE endpoint for mcp usage --- .../common_daily_activity.py | 48 ++++++++++++++++--- .../mcp_management_endpoints.py | 3 ++ .../common_daily_activity.py | 6 ++- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 5efd31b2329..f5b10f0ad89 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -44,16 +44,27 @@ def update_breakdown_metrics( """Updates breakdown metrics for a single record using the existing update_metrics function""" # Update model breakdown - if record.model not in breakdown.models: + if record.model and record.model not in breakdown.models: breakdown.models[record.model] = MetricWithMetadata( metrics=SpendMetrics(), metadata=model_metadata.get( record.model, {} ), # Add any model-specific metadata here ) - breakdown.models[record.model].metrics = update_metrics( - breakdown.models[record.model].metrics, record - ) + if record.model: + breakdown.models[record.model].metrics = update_metrics( + breakdown.models[record.model].metrics, record + ) + + if record.mcp_server_id: + if record.mcp_server_id not in breakdown.mcp_servers: + breakdown.mcp_servers[record.mcp_server_id] = MetricWithMetadata( + metrics=SpendMetrics(), + metadata={}, + ) + breakdown.mcp_servers[record.mcp_server_id].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_server_id].metrics, record + ) # Update provider breakdown provider = record.custom_llm_provider or "unknown" @@ -92,9 +103,11 @@ def update_breakdown_metrics( if entity_value not in breakdown.entities: breakdown.entities[entity_value] = MetricWithMetadata( metrics=SpendMetrics(), - metadata=entity_metadata_field.get(entity_value, {}) - if entity_metadata_field - else {}, + metadata=( + entity_metadata_field.get(entity_value, {}) + if entity_metadata_field + else {} + ), ) breakdown.entities[entity_value].metrics = update_metrics( breakdown.entities[entity_value].metrics, record @@ -131,6 +144,10 @@ async def get_daily_activity( exclude_entity_ids: Optional[List[str]] = None, ) -> SpendAnalyticsPaginatedResponse: """Common function to get daily activity for any entity type.""" + from litellm.types.proxy.management_endpoints.common_daily_activity import ( + LiteLLM_DailyUserSpend, + ) + if prisma_client is None: raise HTTPException( status_code=500, @@ -181,6 +198,23 @@ async def get_daily_activity( take=page_size, ) + # for 50% of the records, set the mcp_server_id to a random value + import random + + for idx, record in enumerate(daily_spend_data): + record = LiteLLM_DailyUserSpend(**record.model_dump()) + if random.random() < 0.5: + record.mcp_server_id = "random_mcp_server_id_" + str( + random.randint(1, 1000000) + ) + record.model = None + record.model_group = None + record.prompt_tokens = 0 + record.completion_tokens = 0 + record.cache_read_input_tokens = 0 + record.cache_creation_input_tokens = 0 + daily_spend_data[idx] = record + # Get all unique API keys from the spend data api_keys = set() for record in daily_spend_data: diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index 3ddd96c1f29..df8dde5188d 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -52,6 +52,9 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_helpers.utils import management_endpoint_wrapper + from litellm.types.proxy.management_endpoints.common_daily_activity import ( + SpendAnalyticsPaginatedResponse, + ) def get_prisma_client_or_throw(message: str): from litellm.proxy.proxy_server import prisma_client diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 6213087f64d..fb6e1dbc99c 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -51,6 +51,9 @@ class KeyMetricWithMetadata(MetricBase): class BreakdownMetrics(BaseModel): """Breakdown of spend by different dimensions""" + mcp_servers: Dict[str, MetricWithMetadata] = Field( + default_factory=dict + ) # mcp_server -> {metrics, metadata} models: Dict[str, MetricWithMetadata] = Field( default_factory=dict ) # model -> {metrics, metadata} @@ -96,7 +99,8 @@ class LiteLLM_DailyUserSpend(BaseModel): user_id: str date: str api_key: str - model: str + mcp_server_id: Optional[str] = None + model: Optional[str] = None model_group: Optional[str] = None custom_llm_provider: Optional[str] = None prompt_tokens: int = 0 From 6c1b72bf3f6dd21551e2dd298c464b90bf164c81 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 15:28:46 -0700 Subject: [PATCH 02/23] feat(ui/): show mcp server activity on UI allows admin to know which mcp's are being used --- ui/litellm-dashboard/src/components/activity_metrics.tsx | 2 +- ui/litellm-dashboard/src/components/new_usage.tsx | 5 +++++ ui/litellm-dashboard/src/components/usage/types.ts | 1 + 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 5b30f290bbe..ee07f7d74b6 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -252,7 +252,7 @@ const formatKeyLabel = (modelData: KeyMetricWithMetadata, model: string): string }; // Process data function -export const processActivityData = (dailyActivity: { results: DailyData[] }, key: "models" | "api_keys"): Record => { +export const processActivityData = (dailyActivity: { results: DailyData[] }, key: "models" | "api_keys" | "mcp_servers"): Record => { const modelMetrics: Record = {}; dailyActivity.results.forEach((day) => { diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index 9269a6586b1..83b4fb3a7d6 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -265,6 +265,7 @@ const NewUsagePage: React.FC = ({ const modelMetrics = processActivityData(userSpendData, "models"); const keyMetrics = processActivityData(userSpendData, "api_keys"); + const mcpServerMetrics = processActivityData(userSpendData, "mcp_servers"); return (
@@ -297,6 +298,7 @@ const NewUsagePage: React.FC = ({ Cost Model Activity Key Activity + MCP Server Activity {/* Cost Panel */} @@ -501,6 +503,9 @@ const NewUsagePage: React.FC = ({ + + + diff --git a/ui/litellm-dashboard/src/components/usage/types.ts b/ui/litellm-dashboard/src/components/usage/types.ts index 46d96b4aa1c..eb55fe056ad 100644 --- a/ui/litellm-dashboard/src/components/usage/types.ts +++ b/ui/litellm-dashboard/src/components/usage/types.ts @@ -84,6 +84,7 @@ export interface MetricWithMetadata { export interface BreakdownMetrics { models: { [key: string]: MetricWithMetadata }; + mcp_servers: { [key: string]: MetricWithMetadata }; providers: { [key: string]: MetricWithMetadata }; api_keys: { [key: string]: KeyMetricWithMetadata }; } From 2e520ed5c2aec1fcf6197a30c26b8e37371ca484 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 15:44:40 -0700 Subject: [PATCH 03/23] feat(common_daily_activity.py): return activity by key --- .../common_daily_activity.py | 97 +++++++++++++++++++ .../common_daily_activity.py | 12 ++- 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index f5b10f0ad89..6e4af9f3912 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -56,6 +56,30 @@ def update_breakdown_metrics( breakdown.models[record.model].metrics, record ) + # Update API key breakdown for this model + if record.api_key not in breakdown.models[record.model].api_key_breakdown: + breakdown.models[record.model].api_key_breakdown[record.api_key] = ( + KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + ) + breakdown.models[record.model].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.models[record.model] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) + ) + if record.mcp_server_id: if record.mcp_server_id not in breakdown.mcp_servers: breakdown.mcp_servers[record.mcp_server_id] = MetricWithMetadata( @@ -66,6 +90,33 @@ def update_breakdown_metrics( breakdown.mcp_servers[record.mcp_server_id].metrics, record ) + # Update API key breakdown for this MCP server + if ( + record.api_key + not in breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown + ): + breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown[ + record.api_key + ] = KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown[ + record.api_key + ].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_server_id] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) + # Update provider breakdown provider = record.custom_llm_provider or "unknown" if provider not in breakdown.providers: @@ -79,6 +130,28 @@ def update_breakdown_metrics( breakdown.providers[provider].metrics, record ) + # Update API key breakdown for this provider + if record.api_key not in breakdown.providers[provider].api_key_breakdown: + breakdown.providers[provider].api_key_breakdown[record.api_key] = ( + KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + ) + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, + record, + ) + ) + # Update api key breakdown if record.api_key not in breakdown.api_keys: breakdown.api_keys[record.api_key] = KeyMetricWithMetadata( @@ -113,6 +186,30 @@ def update_breakdown_metrics( breakdown.entities[entity_value].metrics, record ) + # Update API key breakdown for this entity + if record.api_key not in breakdown.entities[entity_value].api_key_breakdown: + breakdown.entities[entity_value].api_key_breakdown[record.api_key] = ( + KeyMetricWithMetadata( + metrics=SpendMetrics(), + metadata=KeyMetadata( + key_alias=api_key_metadata.get(record.api_key, {}).get( + "key_alias", None + ), + team_id=api_key_metadata.get(record.api_key, {}).get( + "team_id", None + ), + ), + ) + ) + breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.entities[entity_value] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) + ) + return breakdown diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index fb6e1dbc99c..fc87f63fdf0 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -31,10 +31,6 @@ class MetricBase(BaseModel): metrics: SpendMetrics -class MetricWithMetadata(MetricBase): - metadata: Dict[str, Any] = Field(default_factory=dict) - - class KeyMetadata(BaseModel): """Metadata for a key""" @@ -48,6 +44,14 @@ class KeyMetricWithMetadata(MetricBase): metadata: KeyMetadata = Field(default_factory=KeyMetadata) +class MetricWithMetadata(MetricBase): + metadata: Dict[str, Any] = Field(default_factory=dict) + # API key breakdown for this metric (e.g., which API keys are using this MCP server) + api_key_breakdown: Dict[str, KeyMetricWithMetadata] = Field( + default_factory=dict + ) # api_key -> {metrics, metadata} + + class BreakdownMetrics(BaseModel): """Breakdown of spend by different dimensions""" From a6afa31f158c3490e8386e89e07bf9a686d218e7 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 15:54:51 -0700 Subject: [PATCH 04/23] feat(ui/): show top api keys for a given model / mcp server allow user to know which key is driving spend --- .../src/components/activity_metrics.tsx | 67 ++++++++++++++++++- .../src/components/usage/types.ts | 40 ++++++----- 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index ee07f7d74b6..6cd59a9c1cb 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -1,7 +1,7 @@ import React from 'react'; import { Card, Grid, Text, Title, Accordion, AccordionHeader, AccordionBody } from '@tremor/react'; import { AreaChart, BarChart } from '@tremor/react'; -import { SpendMetrics, DailyData, ModelActivityData, MetricWithMetadata, KeyMetricWithMetadata } from './usage/types'; +import { SpendMetrics, DailyData, ModelActivityData, MetricWithMetadata, KeyMetricWithMetadata, TopApiKeyData } from './usage/types'; import { Collapse } from 'antd'; import { formatNumberWithCommas } from '@/utils/dataUtils'; @@ -34,6 +34,35 @@ const ModelSection = ({ modelName, metrics }: { modelName: string; metrics: Mode + {/* Top API Keys Section */} + {metrics.top_api_keys && metrics.top_api_keys.length > 0 && ( + + Top API Keys by Spend +
+
+ {metrics.top_api_keys.map((keyData, index) => ( +
+
+ + {keyData.key_alias || `${keyData.api_key.substring(0, 10)}...`} + + {keyData.team_id && ( + Team: {keyData.team_id} + )} +
+
+ ${formatNumberWithCommas(keyData.spend, 2)} + + {keyData.requests.toLocaleString()} requests | {keyData.tokens.toLocaleString()} tokens + +
+
+ ))} +
+
+
+ )} + {/* Charts */} @@ -271,6 +300,7 @@ export const processActivityData = (dailyActivity: { results: DailyData[] }, key total_spend: 0, total_cache_read_input_tokens: 0, total_cache_creation_input_tokens: 0, + top_api_keys: [], daily_data: [] }; } @@ -304,6 +334,41 @@ export const processActivityData = (dailyActivity: { results: DailyData[] }, key }); }); + // Process API key breakdowns for each metric (skip if key is 'api_keys' to avoid duplication) + if (key !== 'api_keys') { + Object.entries(modelMetrics).forEach(([model, _]) => { + const apiKeyBreakdown: Record = {}; + + // Aggregate API key data across all days + dailyActivity.results.forEach((day) => { + const modelData = day.breakdown[key]?.[model]; + if (modelData && 'api_key_breakdown' in modelData) { + Object.entries(modelData.api_key_breakdown || {}).forEach(([apiKey, keyData]) => { + if (!apiKeyBreakdown[apiKey]) { + apiKeyBreakdown[apiKey] = { + api_key: apiKey, + key_alias: keyData.metadata.key_alias, + team_id: keyData.metadata.team_id, + spend: 0, + requests: 0, + tokens: 0, + }; + } + + apiKeyBreakdown[apiKey].spend += keyData.metrics.spend; + apiKeyBreakdown[apiKey].requests += keyData.metrics.api_requests; + apiKeyBreakdown[apiKey].tokens += keyData.metrics.total_tokens; + }); + } + }); + + // Sort by spend and take top 5 + modelMetrics[model].top_api_keys = Object.values(apiKeyBreakdown) + .sort((a, b) => b.spend - a.spend) + .slice(0, 5); + }); + } + // Sort daily data Object.values(modelMetrics).forEach(metrics => { metrics.daily_data.sort((a, b) => new Date(a.date).getTime() - new Date(b.date).getTime()); diff --git a/ui/litellm-dashboard/src/components/usage/types.ts b/ui/litellm-dashboard/src/components/usage/types.ts index eb55fe056ad..3d76517d9cc 100644 --- a/ui/litellm-dashboard/src/components/usage/types.ts +++ b/ui/litellm-dashboard/src/components/usage/types.ts @@ -18,21 +18,35 @@ export interface DailyData { export interface BreakdownMetrics { models: { [key: string]: MetricWithMetadata }; + mcp_servers: { [key: string]: MetricWithMetadata }; providers: { [key: string]: MetricWithMetadata }; api_keys: { [key: string]: KeyMetricWithMetadata }; + entities: { [key: string]: MetricWithMetadata }; } export interface MetricWithMetadata { metrics: SpendMetrics; metadata: object; + api_key_breakdown: { [key: string]: KeyMetricWithMetadata }; } export interface KeyMetricWithMetadata { metrics: SpendMetrics; - metadata: { - key_alias: string | null; - team_id?: string | null; - }; + metadata: KeyMetadata; +} + +export interface KeyMetadata { + key_alias: string | null; + team_id: string | null; +} + +export interface TopApiKeyData { + api_key: string; + key_alias: string | null; + team_id: string | null; + spend: number; + requests: number; + tokens: number; } export interface ModelActivityData { @@ -46,6 +60,7 @@ export interface ModelActivityData { prompt_tokens: number; completion_tokens: number; total_spend: number; + top_api_keys: TopApiKeyData[]; daily_data: { date: string; metrics: { @@ -62,11 +77,6 @@ export interface ModelActivityData { }[]; } -export interface KeyMetadata { - key_alias: string | null; - team_id: string | null; -} - export interface EntityMetadata { alias: string; id: string; @@ -76,15 +86,3 @@ export interface EntityMetricWithMetadata { metrics: SpendMetrics; metadata: EntityMetadata; } - -export interface MetricWithMetadata { - metrics: SpendMetrics; - metadata: object; -} - -export interface BreakdownMetrics { - models: { [key: string]: MetricWithMetadata }; - mcp_servers: { [key: string]: MetricWithMetadata }; - providers: { [key: string]: MetricWithMetadata }; - api_keys: { [key: string]: KeyMetricWithMetadata }; -} From 929b69b8b98132fd6062e9e296c9ae28b1e16c94 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 16:04:53 -0700 Subject: [PATCH 05/23] fix(common_daily_activity.py): use known mcp server names --- litellm/proxy/management_endpoints/common_daily_activity.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 6e4af9f3912..7b0658e0244 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -296,14 +296,13 @@ async def get_daily_activity( ) # for 50% of the records, set the mcp_server_id to a random value + mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"} import random for idx, record in enumerate(daily_spend_data): record = LiteLLM_DailyUserSpend(**record.model_dump()) if random.random() < 0.5: - record.mcp_server_id = "random_mcp_server_id_" + str( - random.randint(1, 1000000) - ) + record.mcp_server_id = random.choice(list(mcp_server_dict)) record.model = None record.model_group = None record.prompt_tokens = 0 From 2725c46c23f9d609d2d5700b38a435da351d942e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 16:34:09 -0700 Subject: [PATCH 06/23] feat(server.py): log the namespaced tool name (includes server prefix) allow accurate cost tracking --- .../proxy/_experimental/mcp_server/server.py | 62 +++++++++++-------- litellm/proxy/db/db_spend_update_writer.py | 19 +++--- litellm/types/utils.py | 7 +++ 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 930d099f83c..28187018aad 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -223,7 +223,7 @@ async def mcp_server_tool_call( async def _get_tools_from_mcp_servers( user_api_key_auth: Optional[UserAPIKeyAuth], mcp_auth_header: Optional[str], - mcp_servers: Optional[List[str]] + mcp_servers: Optional[List[str]], ) -> List[MCPTool]: """ Helper method to fetch tools from MCP servers based on server filtering criteria. @@ -239,12 +239,19 @@ async def _get_tools_from_mcp_servers( if mcp_servers: # If mcp_servers header is present, only get tools from specified servers tools = [] - for server_id in await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth): + for server_id in await global_mcp_server_manager.get_allowed_mcp_servers( + user_api_key_auth + ): server = global_mcp_server_manager.get_mcp_server_by_id(server_id) - if server and any(normalize_server_name(server.name) == normalize_server_name(s) for s in mcp_servers): - server_tools = await global_mcp_server_manager._get_tools_from_server( - server=server, - mcp_auth_header=mcp_auth_header, + if server and any( + normalize_server_name(server.name) == normalize_server_name(s) + for s in mcp_servers + ): + server_tools = ( + await global_mcp_server_manager._get_tools_from_server( + server=server, + mcp_auth_header=mcp_auth_header, + ) ) tools.extend(server_tools) return tools @@ -285,7 +292,7 @@ async def _list_mcp_tools( tools_from_mcp_servers = await _get_tools_from_mcp_servers( user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers + mcp_servers=mcp_servers, ) verbose_logger.debug("TOOLS FROM MCP SERVERS: %s", tools_from_mcp_servers) @@ -295,15 +302,16 @@ async def _list_mcp_tools( @client async def call_mcp_tool( - name: str, - arguments: Optional[Dict[str, Any]] = None, - user_api_key_auth: Optional[UserAPIKeyAuth] = None, - mcp_auth_header: Optional[str] = None, - **kwargs: Any + name: str, + arguments: Optional[Dict[str, Any]] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, + mcp_auth_header: Optional[str] = None, + **kwargs: Any, ) -> List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]: """ Call a specific tool with the provided arguments (handles prefixed tool names) """ + if arguments is None: raise HTTPException( status_code=400, detail="Request arguments are required" @@ -311,12 +319,14 @@ async def call_mcp_tool( # Remove prefix from tool name for logging and processing original_tool_name, server_name_from_prefix = get_server_name_prefix_tool_mcp( - name) + name + ) standard_logging_mcp_tool_call: StandardLoggingMCPToolCall = ( _get_standard_logging_mcp_tool_call( name=original_tool_name, # Use original name for logging arguments=arguments, + server_name=server_name_from_prefix, ) ) litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get( @@ -326,12 +336,6 @@ async def call_mcp_tool( litellm_logging_obj.model_call_details["mcp_tool_call_metadata"] = ( standard_logging_mcp_tool_call ) - litellm_logging_obj.model_call_details["model"] = ( - f"{MCP_TOOL_NAME_PREFIX}: {standard_logging_mcp_tool_call.get('name') or ''}" - ) - litellm_logging_obj.model_call_details["custom_llm_provider"] = ( - standard_logging_mcp_tool_call.get("mcp_server_name") - ) # Try managed server tool first (pass the full prefixed name) if name in global_mcp_server_manager.tool_name_to_mcp_server_name_mapping: @@ -348,6 +352,7 @@ async def call_mcp_tool( def _get_standard_logging_mcp_tool_call( name: str, arguments: Dict[str, Any], + server_name: Optional[str], ) -> StandardLoggingMCPToolCall: mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) if mcp_server: @@ -357,15 +362,17 @@ def _get_standard_logging_mcp_tool_call( arguments=arguments, mcp_server_name=mcp_info.get("server_name"), mcp_server_logo_url=mcp_info.get("logo_url"), + namespaced_server_name=f"{server_name}/{name}" if server_name else name, ) else: return StandardLoggingMCPToolCall( name=name, arguments=arguments, + namespaced_server_name=f"{server_name}/{name}" if server_name else name, ) async def _handle_managed_mcp_tool( - name: str, + name: str, arguments: Dict[str, Any], user_api_key_auth: Optional[UserAPIKeyAuth] = None, mcp_auth_header: Optional[str] = None, @@ -381,7 +388,7 @@ async def _handle_managed_mcp_tool( return call_tool_result.content # type: ignore[return-value] async def _handle_local_mcp_tool( - name: str, arguments: Dict[str, Any] + name: str, arguments: Dict[str, Any] ) -> List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]: """ Handle tool execution for local registry tools @@ -397,7 +404,6 @@ async def _handle_local_mcp_tool( except Exception as e: return [MCPTextContent(text=f"Error: {str(e)}", type="text")] - async def handle_streamable_http_mcp( scope: Scope, receive: Receive, send: Send ) -> None: @@ -479,7 +485,7 @@ def get_mcp_server_enabled() -> Dict[str, bool]: ######################################################## def set_auth_context( - user_api_key_auth: UserAPIKeyAuth, + user_api_key_auth: UserAPIKeyAuth, mcp_auth_header: Optional[str] = None, mcp_servers: Optional[List[str]] = None, ) -> None: @@ -498,7 +504,9 @@ def set_auth_context( ) auth_context_var.set(auth_user) - def get_auth_context() -> Tuple[Optional[UserAPIKeyAuth], Optional[str], Optional[List[str]]]: + def get_auth_context() -> ( + Tuple[Optional[UserAPIKeyAuth], Optional[str], Optional[List[str]]] + ): """ Get the UserAPIKeyAuth from the auth context variable. @@ -507,7 +515,11 @@ def get_auth_context() -> Tuple[Optional[UserAPIKeyAuth], Optional[str], Optiona """ auth_user = auth_context_var.get() if auth_user and isinstance(auth_user, MCPAuthenticatedUser): - return auth_user.user_api_key_auth, auth_user.mcp_auth_header, auth_user.mcp_servers + return ( + auth_user.user_api_key_auth, + auth_user.mcp_auth_header, + auth_user.mcp_servers, + ) return None, None, None ######################################################## diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 9a7b14272da..588f4f2344b 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -775,6 +775,8 @@ async def _commit_spend_updates_to_db( # noqa: PLR0915 e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj ) + # fmt: off + @overload @staticmethod async def _update_daily_spend( @@ -786,7 +788,7 @@ async def _update_daily_spend( entity_id_field: str, table_name: str, unique_constraint_name: str, - ) -> None: + ) -> None: ... @overload @@ -814,8 +816,9 @@ async def _update_daily_spend( entity_id_field: str, table_name: str, unique_constraint_name: str, - ) -> None: + ) -> None: ... + # fmt: on @staticmethod async def _update_daily_spend( @@ -898,13 +901,13 @@ async def _update_daily_spend( # Add cache-related fields if they exist if "cache_read_input_tokens" in transaction: - common_data[ - "cache_read_input_tokens" - ] = transaction.get("cache_read_input_tokens", 0) + common_data["cache_read_input_tokens"] = ( + transaction.get("cache_read_input_tokens", 0) + ) if "cache_creation_input_tokens" in transaction: - common_data[ - "cache_creation_input_tokens" - ] = transaction.get("cache_creation_input_tokens", 0) + common_data["cache_creation_input_tokens"] = ( + transaction.get("cache_creation_input_tokens", 0) + ) # Create update data structure update_data = { diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4ff678d329c..adb65da40cb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1841,6 +1841,13 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): (this is to render the logo on the logs page on litellm ui) """ + namespaced_server_name: Optional[str] + """ + Namespaced server name of the MCP server that the tool call was made to + + Includes the server name prefix if it exists + """ + class StandardLoggingVectorStoreRequest(TypedDict, total=False): """ From 0e0efc2220cfa9891d5bc8048bd2996948a0d55f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 17:11:35 -0700 Subject: [PATCH 07/23] feat(db_spend_update_writer.py): log by mcp_namespaced_tool_name store aggregate daily activity by mcp_namespaced_tool_name Enables cost / usage tracking by mcp tool name --- .../proxy/_experimental/mcp_server/server.py | 4 +- litellm/proxy/_types.py | 2 + litellm/proxy/db/db_spend_update_writer.py | 40 ++++++++++++++----- .../common_daily_activity.py | 30 +++++++------- litellm/proxy/schema.prisma | 25 +++++++----- .../spend_tracking/spend_tracking_utils.py | 8 ++++ litellm/types/utils.py | 6 +-- 7 files changed, 77 insertions(+), 38 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 28187018aad..d6e5a691609 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -362,13 +362,13 @@ def _get_standard_logging_mcp_tool_call( arguments=arguments, mcp_server_name=mcp_info.get("server_name"), mcp_server_logo_url=mcp_info.get("logo_url"), - namespaced_server_name=f"{server_name}/{name}" if server_name else name, + namespaced_tool_name=f"{server_name}/{name}" if server_name else name, ) else: return StandardLoggingMCPToolCall( name=name, arguments=arguments, - namespaced_server_name=f"{server_name}/{name}" if server_name else name, + namespaced_tool_name=f"{server_name}/{name}" if server_name else name, ) async def _handle_managed_mcp_tool( diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index aa2f3b7a3a8..6852a99c501 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2173,6 +2173,7 @@ class SpendLogsPayload(TypedDict): model: str model_id: Optional[str] model_group: Optional[str] + mcp_namespaced_tool_name: Optional[str] api_base: str user: str metadata: str # json str @@ -3055,6 +3056,7 @@ class BaseDailySpendTransaction(TypedDict): api_key: str model: str model_group: Optional[str] + mcp_namespaced_tool_name: Optional[str] custom_llm_provider: Optional[str] # token count metrics diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 588f4f2344b..a5398fa7f2f 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -873,6 +873,9 @@ async def _update_daily_spend( "custom_llm_provider": transaction.get( "custom_llm_provider" ), + "mcp_namespaced_tool_name": transaction.get( + "mcp_namespaced_tool_name" + ), } } @@ -884,8 +887,11 @@ async def _update_daily_spend( entity_id_field: entity_id, "date": transaction["date"], "api_key": transaction["api_key"], - "model": transaction["model"], + "model": transaction.get("model"), "model_group": transaction.get("model_group"), + "mcp_namespaced_tool_name": transaction.get( + "mcp_namespaced_tool_name" + ), "custom_llm_provider": transaction.get( "custom_llm_provider" ), @@ -996,7 +1002,7 @@ async def update_daily_user_spend( entity_type="user", entity_id_field="user_id", table_name="litellm_dailyuserspend", - unique_constraint_name="user_id_date_api_key_model_custom_llm_provider", + unique_constraint_name="user_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", ) @staticmethod @@ -1017,7 +1023,7 @@ async def update_daily_team_spend( entity_type="team", entity_id_field="team_id", table_name="litellm_dailyteamspend", - unique_constraint_name="team_id_date_api_key_model_custom_llm_provider", + unique_constraint_name="team_id_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", ) @staticmethod @@ -1038,7 +1044,7 @@ async def update_daily_tag_spend( entity_type="tag", entity_id_field="tag", table_name="litellm_dailytagspend", - unique_constraint_name="tag_date_api_key_model_custom_llm_provider", + unique_constraint_name="tag_date_api_key_model_custom_llm_provider_mcp_namespaced_tool_name", ) async def _common_add_spend_log_transaction_to_daily_transaction( @@ -1047,7 +1053,7 @@ async def _common_add_spend_log_transaction_to_daily_transaction( prisma_client: PrismaClient, type: Literal["user", "team", "request_tags"] = "user", ) -> Optional[BaseDailySpendTransaction]: - common_expected_keys = ["startTime", "api_key", "model", "custom_llm_provider"] + common_expected_keys = ["startTime", "api_key"] if type == "user": expected_keys = ["user", *common_expected_keys] elif type == "team": @@ -1056,13 +1062,28 @@ async def _common_add_spend_log_transaction_to_daily_transaction( expected_keys = ["request_tags", *common_expected_keys] else: raise ValueError(f"Invalid type: {type}") - if not all(key in payload for key in expected_keys): verbose_proxy_logger.debug( f"Missing expected keys: {expected_keys}, in payload, skipping from daily_user_spend_transactions" ) return None + any_expected_keys = ["model", "mcp_namespaced_tool_name"] + if not any(key in payload for key in any_expected_keys): + verbose_proxy_logger.debug( + f"Missing any expected keys: {any_expected_keys}, in payload, skipping from daily_user_spend_transactions" + ) + return None + elif "mcp_namespaced_tool_name" in payload: + pass + elif "model" in payload and ( + "custom_llm_provider" not in payload or "model_group" not in payload + ): + verbose_proxy_logger.debug( + "Missing custom_llm_provider or model_group in payload, skipping from daily_user_spend_transactions" + ) + return None + request_status = prisma_client.get_request_status(payload) verbose_proxy_logger.info(f"Logged request status: {request_status}") _metadata: SpendLogsMetadata = json.loads(payload["metadata"]) @@ -1081,9 +1102,10 @@ async def _common_add_spend_log_transaction_to_daily_transaction( daily_transaction = BaseDailySpendTransaction( date=date, api_key=payload["api_key"], - model=payload["model"], - model_group=payload["model_group"], - custom_llm_provider=payload["custom_llm_provider"], + model=payload.get("model", None), + model_group=payload.get("model_group", None), + mcp_namespaced_tool_name=payload.get("mcp_namespaced_tool_name", None), + custom_llm_provider=payload.get("custom_llm_provider", None), prompt_tokens=payload["prompt_tokens"], completion_tokens=payload["completion_tokens"], spend=payload["spend"], diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 7b0658e0244..16486fabd20 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -295,21 +295,21 @@ async def get_daily_activity( take=page_size, ) - # for 50% of the records, set the mcp_server_id to a random value - mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"} - import random - - for idx, record in enumerate(daily_spend_data): - record = LiteLLM_DailyUserSpend(**record.model_dump()) - if random.random() < 0.5: - record.mcp_server_id = random.choice(list(mcp_server_dict)) - record.model = None - record.model_group = None - record.prompt_tokens = 0 - record.completion_tokens = 0 - record.cache_read_input_tokens = 0 - record.cache_creation_input_tokens = 0 - daily_spend_data[idx] = record + # # for 50% of the records, set the mcp_server_id to a random value + # mcp_server_dict = {"Zapier_Gmail_MCP", "Stripe_MCP"} + # import random + + # for idx, record in enumerate(daily_spend_data): + # record = LiteLLM_DailyUserSpend(**record.model_dump()) + # if random.random() < 0.5: + # record.mcp_server_id = random.choice(list(mcp_server_dict)) + # record.model = None + # record.model_group = None + # record.prompt_tokens = 0 + # record.completion_tokens = 0 + # record.cache_read_input_tokens = 0 + # record.cache_creation_input_tokens = 0 + # daily_spend_data[idx] = record # Get all unique API keys from the spend data api_keys = set() diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 9b0fbbaa8f2..8b6aa7481fc 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -261,6 +261,7 @@ model LiteLLM_SpendLogs { response Json? @default("{}") session_id String? status String? + mcp_namespaced_tool_name String? proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) @@ -359,9 +360,10 @@ model LiteLLM_DailyUserSpend { user_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -373,11 +375,12 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -386,9 +389,10 @@ model LiteLLM_DailyTeamSpend { team_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -400,11 +404,12 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -413,9 +418,10 @@ model LiteLLM_DailyTagSpend { tag String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -427,11 +433,12 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 98ec274380f..9f2c7772e80 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -286,6 +286,13 @@ def get_logging_payload( # noqa: PLR0915 id = f"{id}_cache_hit{time.time()}" # SpendLogs does not allow duplicate request_id + mcp_namespaced_tool_name = None + mcp_tool_call_metadata = clean_metadata.get("mcp_tool_call_metadata", {}) + if mcp_tool_call_metadata is not None: + mcp_namespaced_tool_name = mcp_tool_call_metadata.get( + "namespaced_tool_name", None + ) + try: payload: SpendLogsPayload = SpendLogsPayload( request_id=str(id), @@ -311,6 +318,7 @@ def get_logging_payload( # noqa: PLR0915 api_base=litellm_params.get("api_base", ""), model_group=_model_group, model_id=_model_id, + mcp_namespaced_tool_name=mcp_namespaced_tool_name, requester_ip_address=clean_metadata.get("requester_ip_address", None), custom_llm_provider=kwargs.get("custom_llm_provider", ""), messages=_get_messages_for_spend_logs_payload( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index adb65da40cb..1c65e9dd577 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1841,11 +1841,11 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): (this is to render the logo on the logs page on litellm ui) """ - namespaced_server_name: Optional[str] + namespaced_tool_name: Optional[str] """ - Namespaced server name of the MCP server that the tool call was made to + Namespaced tool name of the MCP tool that the tool call was made to - Includes the server name prefix if it exists + Includes the server name prefix if it exists - eg. `deepwiki-mcp/get_page_content` """ From cacd89c803e0656c66554393f7e736e0dd4ebcf5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 21:48:20 -0700 Subject: [PATCH 08/23] fix(server.py): add key/user metadata to mcp calls --- .../proxy/_experimental/mcp_server/server.py | 42 ++++++++++++++++--- 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index d6e5a691609..85ab21f78f6 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -199,17 +199,47 @@ async def mcp_server_tool_call( Raises: HTTPException: If tool not found or arguments missing """ + from fastapi import Request + + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import proxy_config + # Validate arguments user_api_key_auth, mcp_auth_header, _ = get_auth_context() + verbose_logger.debug( f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" ) - response = await call_mcp_tool( - name=name, - arguments=arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - ) + try: + request = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/mcp/tools/call", + "headers": {}, + } + ) + if user_api_key_auth is not None: + data = await add_litellm_data_to_request( + data={}, + request=request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, + ) + else: + data = {} + + response = await call_mcp_tool( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + **data, # for logging + ) + except Exception as e: + verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") + raise e + return response ######################################################## From faec6cc99035388fed9f0d5df59b2cbaad2c2cbf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 21:20:49 -0700 Subject: [PATCH 09/23] refactor(common_daily_activity.py): update to return mcp activity in API --- .../common_daily_activity.py | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 16486fabd20..767edde0cde 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -57,7 +57,10 @@ def update_breakdown_metrics( ) # Update API key breakdown for this model - if record.api_key not in breakdown.models[record.model].api_key_breakdown: + if ( + record.api_key + and record.api_key not in breakdown.models[record.model].api_key_breakdown + ): breakdown.models[record.model].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -80,22 +83,25 @@ def update_breakdown_metrics( ) ) - if record.mcp_server_id: - if record.mcp_server_id not in breakdown.mcp_servers: - breakdown.mcp_servers[record.mcp_server_id] = MetricWithMetadata( + if record.mcp_namespaced_tool_name: + if record.mcp_namespaced_tool_name not in breakdown.mcp_servers: + breakdown.mcp_servers[record.mcp_namespaced_tool_name] = MetricWithMetadata( metrics=SpendMetrics(), metadata={}, ) - breakdown.mcp_servers[record.mcp_server_id].metrics = update_metrics( - breakdown.mcp_servers[record.mcp_server_id].metrics, record + breakdown.mcp_servers[record.mcp_namespaced_tool_name].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_namespaced_tool_name].metrics, record ) # Update API key breakdown for this MCP server if ( record.api_key - not in breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown + and record.api_key + not in breakdown.mcp_servers[ + record.mcp_namespaced_tool_name + ].api_key_breakdown ): - breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown[ + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ record.api_key ] = KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -108,10 +114,10 @@ def update_breakdown_metrics( ), ), ) - breakdown.mcp_servers[record.mcp_server_id].api_key_breakdown[ + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ record.api_key ].metrics = update_metrics( - breakdown.mcp_servers[record.mcp_server_id] + breakdown.mcp_servers[record.mcp_namespaced_tool_name] .api_key_breakdown[record.api_key] .metrics, record, @@ -131,7 +137,10 @@ def update_breakdown_metrics( ) # Update API key breakdown for this provider - if record.api_key not in breakdown.providers[provider].api_key_breakdown: + if ( + record.api_key + and record.api_key not in breakdown.providers[provider].api_key_breakdown + ): breakdown.providers[provider].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -187,7 +196,10 @@ def update_breakdown_metrics( ) # Update API key breakdown for this entity - if record.api_key not in breakdown.entities[entity_value].api_key_breakdown: + if ( + record.api_key + and record.api_key not in breakdown.entities[entity_value].api_key_breakdown + ): breakdown.entities[entity_value].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), From 99cf1a2f8499f56e926f9367c2d328b0207494c3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 21:26:49 -0700 Subject: [PATCH 10/23] fix(common_daily_activity.py): handle empty key --- .../common_daily_activity.py | 60 ++++++++++--------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 767edde0cde..341fd9ef0b4 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -74,14 +74,15 @@ def update_breakdown_metrics( ), ) ) - breakdown.models[record.model].api_key_breakdown[record.api_key].metrics = ( - update_metrics( - breakdown.models[record.model] - .api_key_breakdown[record.api_key] - .metrics, - record, + if record.api_key: + breakdown.models[record.model].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.models[record.model] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) ) - ) if record.mcp_namespaced_tool_name: if record.mcp_namespaced_tool_name not in breakdown.mcp_servers: @@ -114,14 +115,15 @@ def update_breakdown_metrics( ), ), ) - breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ - record.api_key - ].metrics = update_metrics( - breakdown.mcp_servers[record.mcp_namespaced_tool_name] - .api_key_breakdown[record.api_key] - .metrics, - record, - ) + if record.api_key: + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ + record.api_key + ].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_namespaced_tool_name] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) # Update provider breakdown provider = record.custom_llm_provider or "unknown" @@ -154,12 +156,13 @@ def update_breakdown_metrics( ), ) ) - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = ( - update_metrics( - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, - record, + if record.api_key: + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, + record, + ) ) - ) # Update api key breakdown if record.api_key not in breakdown.api_keys: @@ -172,9 +175,10 @@ def update_breakdown_metrics( team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), ), # Add any api_key-specific metadata here ) - breakdown.api_keys[record.api_key].metrics = update_metrics( - breakdown.api_keys[record.api_key].metrics, record - ) + if record.api_key: + breakdown.api_keys[record.api_key].metrics = update_metrics( + breakdown.api_keys[record.api_key].metrics, record + ) # Update entity-specific metrics if entity_id_field is provided if entity_id_field: @@ -213,14 +217,15 @@ def update_breakdown_metrics( ), ) ) - breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = ( - update_metrics( + if record.api_key: + breakdown.entities[entity_value].api_key_breakdown[ + record.api_key + ].metrics = update_metrics( breakdown.entities[entity_value] .api_key_breakdown[record.api_key] .metrics, record, ) - ) return breakdown @@ -253,9 +258,6 @@ async def get_daily_activity( exclude_entity_ids: Optional[List[str]] = None, ) -> SpendAnalyticsPaginatedResponse: """Common function to get daily activity for any entity type.""" - from litellm.types.proxy.management_endpoints.common_daily_activity import ( - LiteLLM_DailyUserSpend, - ) if prisma_client is None: raise HTTPException( From ec938473686ffddc624624e242a94f1f0a63feff Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:05:33 -0700 Subject: [PATCH 11/23] fix(common_daily_activity.py): track when api key is empty --- .../common_daily_activity.py | 74 ++++++++----------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py index 341fd9ef0b4..754169a8197 100644 --- a/litellm/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/proxy/management_endpoints/common_daily_activity.py @@ -57,10 +57,7 @@ def update_breakdown_metrics( ) # Update API key breakdown for this model - if ( - record.api_key - and record.api_key not in breakdown.models[record.model].api_key_breakdown - ): + if record.api_key not in breakdown.models[record.model].api_key_breakdown: breakdown.models[record.model].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -74,15 +71,14 @@ def update_breakdown_metrics( ), ) ) - if record.api_key: - breakdown.models[record.model].api_key_breakdown[record.api_key].metrics = ( - update_metrics( - breakdown.models[record.model] - .api_key_breakdown[record.api_key] - .metrics, - record, - ) + breakdown.models[record.model].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.models[record.model] + .api_key_breakdown[record.api_key] + .metrics, + record, ) + ) if record.mcp_namespaced_tool_name: if record.mcp_namespaced_tool_name not in breakdown.mcp_servers: @@ -97,7 +93,6 @@ def update_breakdown_metrics( # Update API key breakdown for this MCP server if ( record.api_key - and record.api_key not in breakdown.mcp_servers[ record.mcp_namespaced_tool_name ].api_key_breakdown @@ -115,15 +110,15 @@ def update_breakdown_metrics( ), ), ) - if record.api_key: - breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ - record.api_key - ].metrics = update_metrics( - breakdown.mcp_servers[record.mcp_namespaced_tool_name] - .api_key_breakdown[record.api_key] - .metrics, - record, - ) + + breakdown.mcp_servers[record.mcp_namespaced_tool_name].api_key_breakdown[ + record.api_key + ].metrics = update_metrics( + breakdown.mcp_servers[record.mcp_namespaced_tool_name] + .api_key_breakdown[record.api_key] + .metrics, + record, + ) # Update provider breakdown provider = record.custom_llm_provider or "unknown" @@ -139,10 +134,7 @@ def update_breakdown_metrics( ) # Update API key breakdown for this provider - if ( - record.api_key - and record.api_key not in breakdown.providers[provider].api_key_breakdown - ): + if record.api_key not in breakdown.providers[provider].api_key_breakdown: breakdown.providers[provider].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -156,13 +148,12 @@ def update_breakdown_metrics( ), ) ) - if record.api_key: - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = ( - update_metrics( - breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, - record, - ) + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics = ( + update_metrics( + breakdown.providers[provider].api_key_breakdown[record.api_key].metrics, + record, ) + ) # Update api key breakdown if record.api_key not in breakdown.api_keys: @@ -175,10 +166,9 @@ def update_breakdown_metrics( team_id=api_key_metadata.get(record.api_key, {}).get("team_id", None), ), # Add any api_key-specific metadata here ) - if record.api_key: - breakdown.api_keys[record.api_key].metrics = update_metrics( - breakdown.api_keys[record.api_key].metrics, record - ) + breakdown.api_keys[record.api_key].metrics = update_metrics( + breakdown.api_keys[record.api_key].metrics, record + ) # Update entity-specific metrics if entity_id_field is provided if entity_id_field: @@ -200,10 +190,7 @@ def update_breakdown_metrics( ) # Update API key breakdown for this entity - if ( - record.api_key - and record.api_key not in breakdown.entities[entity_value].api_key_breakdown - ): + if record.api_key not in breakdown.entities[entity_value].api_key_breakdown: breakdown.entities[entity_value].api_key_breakdown[record.api_key] = ( KeyMetricWithMetadata( metrics=SpendMetrics(), @@ -217,15 +204,14 @@ def update_breakdown_metrics( ), ) ) - if record.api_key: - breakdown.entities[entity_value].api_key_breakdown[ - record.api_key - ].metrics = update_metrics( + breakdown.entities[entity_value].api_key_breakdown[record.api_key].metrics = ( + update_metrics( breakdown.entities[entity_value] .api_key_breakdown[record.api_key] .metrics, record, ) + ) return breakdown From aabef512499d28bf133f295040d1d00f09acb6be Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:20:17 -0700 Subject: [PATCH 12/23] test(test_spend_management_endpoints.py): update tests --- .../test_spend_management_endpoints.py | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py index 2af95883377..93be9717e96 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py @@ -763,6 +763,7 @@ async def test_spend_logs_payload_e2e(self): "response": "{}", "proxy_server_request": "{}", "status": "success", + "mcp_namespaced_tool_name": None, } ) @@ -855,6 +856,7 @@ async def test_spend_logs_payload_success_log_with_api_base(self, monkeypatch): "response": "{}", "proxy_server_request": "{}", "status": "success", + "mcp_namespaced_tool_name": None, } ) @@ -945,9 +947,13 @@ async def test_spend_logs_payload_success_log_with_router(self): "response": "{}", "proxy_server_request": "{}", "status": "success", + "mcp_namespaced_tool_name": None, } ) + print(f"payload: {payload}") + print(f"expected_payload: {expected_payload}") + differences = _compare_nested_dicts( payload, expected_payload, ignore_keys=ignored_keys ) @@ -1085,8 +1091,8 @@ async def test_global_spend_keys_endpoint_limit_validation(client, monkeypatch): async def test_view_spend_logs_summarize_parameter(client, monkeypatch): """Test the new summarize parameter in the /spend/logs endpoint""" import datetime - from datetime import timezone, timedelta - + from datetime import timedelta, timezone + # Mock spend logs data mock_spend_logs = [ { @@ -1096,7 +1102,9 @@ async def test_view_spend_logs_summarize_parameter(client, monkeypatch): "user": "test_user_1", "team_id": "team1", "spend": 0.05, - "startTime": (datetime.datetime.now(timezone.utc) - timedelta(days=1)).isoformat(), + "startTime": ( + datetime.datetime.now(timezone.utc) - timedelta(days=1) + ).isoformat(), "model": "gpt-3.5-turbo", "prompt_tokens": 100, "completion_tokens": 50, @@ -1109,23 +1117,25 @@ async def test_view_spend_logs_summarize_parameter(client, monkeypatch): "user": "test_user_1", "team_id": "team1", "spend": 0.10, - "startTime": (datetime.datetime.now(timezone.utc) - timedelta(days=1)).isoformat(), + "startTime": ( + datetime.datetime.now(timezone.utc) - timedelta(days=1) + ).isoformat(), "model": "gpt-4", "prompt_tokens": 200, "completion_tokens": 100, "total_tokens": 300, }, ] - + # Mock for unsummarized data (summarize=false) class MockDB: def __init__(self): self.litellm_spendlogs = self - + async def find_many(self, *args, **kwargs): # Return individual log entries when summarize=false return mock_spend_logs - + async def group_by(self, *args, **kwargs): # Return grouped data when summarize=true # Simplified mock response for grouped data @@ -1134,17 +1144,17 @@ async def group_by(self, *args, **kwargs): { "api_key": "sk-test-key", "user": "test_user_1", - "model": "gpt-3.5-turbo", + "model": "gpt-3.5-turbo", "startTime": yesterday.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), - "_sum": {"spend": 0.05} + "_sum": {"spend": 0.05}, }, { "api_key": "sk-test-key", - "user": "test_user_1", + "user": "test_user_1", "model": "gpt-4", "startTime": yesterday.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), - "_sum": {"spend": 0.10} - } + "_sum": {"spend": 0.10}, + }, ] class MockPrismaClient: @@ -1156,7 +1166,9 @@ def __init__(self): monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) # Set up test dates - start_date = (datetime.datetime.now(timezone.utc) - timedelta(days=2)).strftime("%Y-%m-%d") + start_date = (datetime.datetime.now(timezone.utc) - timedelta(days=2)).strftime( + "%Y-%m-%d" + ) end_date = datetime.datetime.now(timezone.utc).strftime("%Y-%m-%d") # Test 1: summarize=false should return individual log entries @@ -1169,10 +1181,10 @@ def __init__(self): }, headers={"Authorization": "Bearer sk-test"}, ) - + assert response.status_code == 200 data = response.json() - + # Should return the raw log entries assert isinstance(data, list) assert len(data) == 2 @@ -1180,7 +1192,7 @@ def __init__(self): assert data[1]["id"] == "log2" assert data[0]["request_id"] == "req1" assert data[1]["request_id"] == "req2" - + # Test 2: summarize=true should return grouped data response = client.get( "/spend/logs", @@ -1191,10 +1203,10 @@ def __init__(self): }, headers={"Authorization": "Bearer sk-test"}, ) - + assert response.status_code == 200 data = response.json() - + # Should return grouped/summarized data assert isinstance(data, list) # The structure should be different - grouped by date with aggregated spend @@ -1202,7 +1214,7 @@ def __init__(self): assert "spend" in data[0] assert "users" in data[0] assert "models" in data[0] - + # Test 3: default behavior (no summarize parameter) should maintain backward compatibility response = client.get( "/spend/logs", @@ -1212,10 +1224,10 @@ def __init__(self): }, headers={"Authorization": "Bearer sk-test"}, ) - + assert response.status_code == 200 data = response.json() - + # Should return grouped/summarized data (same as summarize=true) assert isinstance(data, list) assert "startTime" in data[0] From f08a8bd4ecc77eee43705bf7880f35ebf5eeb3ba Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:27:53 -0700 Subject: [PATCH 13/23] fix: fix ui linting error --- .../src/components/entity_usage.tsx | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ui/litellm-dashboard/src/components/entity_usage.tsx b/ui/litellm-dashboard/src/components/entity_usage.tsx index af66ea39fff..513d63ea932 100644 --- a/ui/litellm-dashboard/src/components/entity_usage.tsx +++ b/ui/litellm-dashboard/src/components/entity_usage.tsx @@ -9,7 +9,7 @@ import { import UsageDatePicker from "./shared/usage_date_picker"; import { Select } from 'antd'; import { ActivityMetrics, processActivityData } from './activity_metrics'; -import { DailyData, KeyMetricWithMetadata, EntityMetricWithMetadata } from './usage/types'; +import { DailyData, BreakdownMetrics, KeyMetricWithMetadata, EntityMetricWithMetadata } from './usage/types'; import { tagDailyActivityCall, teamDailyActivityCall } from './networking'; import TopKeyView from "./top_key_view"; import { formatNumberWithCommas } from "@/utils/dataUtils"; @@ -29,13 +29,6 @@ interface EntityMetrics { metadata: Record; } -interface BreakdownMetrics { - models: Record; - providers: Record; - api_keys: Record; - entities: Record; -} - interface ExtendedDailyData extends DailyData { breakdown: BreakdownMetrics; } @@ -179,7 +172,8 @@ const EntityUsage: React.FC = ({ cache_creation_input_tokens: 0 }, metadata: { - key_alias: metrics.metadata.key_alias + key_alias: metrics.metadata.key_alias, + team_id: metrics.metadata.team_id || null } }; } @@ -265,7 +259,7 @@ const EntityUsage: React.FC = ({ cache_creation_input_tokens: 0 }, metadata: { - alias: data.metadata.team_alias || entity, + alias: (data.metadata as any).team_alias || entity, id: entity } }; From 8e5a38688a8451acc67a162621f1beff4636e989 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:31:52 -0700 Subject: [PATCH 14/23] fix: fix linting errors --- litellm/proxy/_types.py | 2 +- .../internal_user_endpoints.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 6852a99c501..3733939c022 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3054,7 +3054,7 @@ class DefaultInternalUserParams(LiteLLMPydanticObjectBase): class BaseDailySpendTransaction(TypedDict): date: str api_key: str - model: str + model: Optional[str] model_group: Optional[str] mcp_namespaced_tool_name: Optional[str] custom_llm_provider: Optional[str] diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 15cb3d2ad19..1549bd9e20d 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1451,16 +1451,17 @@ def update_breakdown_metrics( """Updates breakdown metrics for a single record using the existing update_metrics function""" # Update model breakdown - if record.model not in breakdown.models: - breakdown.models[record.model] = MetricWithMetadata( - metrics=SpendMetrics(), - metadata=model_metadata.get( - record.model, {} - ), # Add any model-specific metadata here + if record.model: + if record.model not in breakdown.models: + breakdown.models[record.model] = MetricWithMetadata( + metrics=SpendMetrics(), + metadata=model_metadata.get( + record.model, {} + ), # Add any model-specific metadata here + ) + breakdown.models[record.model].metrics = update_metrics( + breakdown.models[record.model].metrics, record ) - breakdown.models[record.model].metrics = update_metrics( - breakdown.models[record.model].metrics, record - ) # Update provider breakdown provider = record.custom_llm_provider or "unknown" From 986f98fc76fadaa4ed0fd2760cdf63080cdd2cce Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:46:43 -0700 Subject: [PATCH 15/23] test: add missing key --- .../gcs_pub_sub_body/spend_logs_payload.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json index 0ee27de8086..b25080df0d0 100644 --- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json +++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json @@ -26,5 +26,6 @@ "messages": "{}", "response": "{}", "proxy_server_request": "{}", - "status": "success" + "status": "success", + "mcp_namespaced_tool_name": null } \ No newline at end of file From 84bb228fab1f454972aa913e6fefa13d141ef04f Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 22:59:46 -0700 Subject: [PATCH 16/23] build(schema.prisma): add mcp tool tracking --- .../litellm_proxy_extras/schema.prisma | 25 ++++++++++++------- schema.prisma | 25 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 9b0fbbaa8f2..8b6aa7481fc 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -261,6 +261,7 @@ model LiteLLM_SpendLogs { response Json? @default("{}") session_id String? status String? + mcp_namespaced_tool_name String? proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) @@ -359,9 +360,10 @@ model LiteLLM_DailyUserSpend { user_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -373,11 +375,12 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -386,9 +389,10 @@ model LiteLLM_DailyTeamSpend { team_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -400,11 +404,12 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -413,9 +418,10 @@ model LiteLLM_DailyTagSpend { tag String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -427,11 +433,12 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } diff --git a/schema.prisma b/schema.prisma index 9b0fbbaa8f2..8b6aa7481fc 100644 --- a/schema.prisma +++ b/schema.prisma @@ -261,6 +261,7 @@ model LiteLLM_SpendLogs { response Json? @default("{}") session_id String? status String? + mcp_namespaced_tool_name String? proxy_server_request Json? @default("{}") @@index([startTime]) @@index([end_user]) @@ -359,9 +360,10 @@ model LiteLLM_DailyUserSpend { user_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -373,11 +375,12 @@ model LiteLLM_DailyUserSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([user_id, date, api_key, model, custom_llm_provider]) + @@unique([user_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([user_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -386,9 +389,10 @@ model LiteLLM_DailyTeamSpend { team_id String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -400,11 +404,12 @@ model LiteLLM_DailyTeamSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([team_id, date, api_key, model, custom_llm_provider]) + @@unique([team_id, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([team_id]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } // Track daily team spend metrics per model and key @@ -413,9 +418,10 @@ model LiteLLM_DailyTagSpend { tag String? date String api_key String - model String + model String? model_group String? - custom_llm_provider String? + custom_llm_provider String? + mcp_namespaced_tool_name String? prompt_tokens BigInt @default(0) completion_tokens BigInt @default(0) cache_read_input_tokens BigInt @default(0) @@ -427,11 +433,12 @@ model LiteLLM_DailyTagSpend { created_at DateTime @default(now()) updated_at DateTime @updatedAt - @@unique([tag, date, api_key, model, custom_llm_provider]) + @@unique([tag, date, api_key, model, custom_llm_provider, mcp_namespaced_tool_name]) @@index([date]) @@index([tag]) @@index([api_key]) @@index([model]) + @@index([mcp_namespaced_tool_name]) } From 5fe2d7fc139b74e4a039b858674d691b616df256 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Mon, 7 Jul 2025 23:00:52 -0700 Subject: [PATCH 17/23] fix(migration.sql): add schema migration file --- .../migration.sql | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20250707230009_add_mcp_namespaced_tool_name/migration.sql diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250707230009_add_mcp_namespaced_tool_name/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250707230009_add_mcp_namespaced_tool_name/migration.sql new file mode 100644 index 00000000000..3130619a773 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250707230009_add_mcp_namespaced_tool_name/migration.sql @@ -0,0 +1,42 @@ +-- DropIndex +DROP INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key"; + +-- DropIndex +DROP INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key"; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTagSpend" ADD COLUMN "mcp_namespaced_tool_name" TEXT, +ALTER COLUMN "model" DROP NOT NULL; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN "mcp_namespaced_tool_name" TEXT, +ALTER COLUMN "model" DROP NOT NULL; + +-- AlterTable +ALTER TABLE "LiteLLM_DailyUserSpend" ADD COLUMN "mcp_namespaced_tool_name" TEXT, +ALTER COLUMN "model" DROP NOT NULL; + +-- AlterTable +ALTER TABLE "LiteLLM_SpendLogs" ADD COLUMN "mcp_namespaced_tool_name" TEXT; + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyTagSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyTagSpend"("mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyTagSpend_tag_date_api_key_model_custom_llm_pro_key" ON "LiteLLM_DailyTagSpend"("tag", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyTeamSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyTeamSpend"("mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyTeamSpend_team_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyTeamSpend"("team_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE INDEX "LiteLLM_DailyUserSpend_mcp_namespaced_tool_name_idx" ON "LiteLLM_DailyUserSpend"("mcp_namespaced_tool_name"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_DailyUserSpend_user_id_date_api_key_model_custom_ll_key" ON "LiteLLM_DailyUserSpend"("user_id", "date", "api_key", "model", "custom_llm_provider", "mcp_namespaced_tool_name"); + From ee967ef88fb666b4210d8656e894ea79d14a0bf4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 18:34:01 -0700 Subject: [PATCH 18/23] feat(server.py): add request logging for mcp calls enables storing the mcp calls --- .../proxy/_experimental/mcp_server/server.py | 10 +- .../mcp_server/test_mcp_server.py | 131 ++++++++++++++++++ 2 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 85ab21f78f6..93871fe5d01 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -4,6 +4,7 @@ import asyncio import contextlib +import json from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union from fastapi import FastAPI, HTTPException @@ -211,17 +212,20 @@ async def mcp_server_tool_call( f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" ) try: + # Create a body date for logging + body_data = {"name": name, "arguments": arguments} + request = Request( scope={ "type": "http", "method": "POST", "path": "/mcp/tools/call", - "headers": {}, + "headers": [(b"content-type", b"application/json")], } ) if user_api_key_auth is not None: data = await add_litellm_data_to_request( - data={}, + data=body_data, request=request, user_api_key_dict=user_api_key_auth, proxy_config=proxy_config, @@ -230,8 +234,6 @@ async def mcp_server_tool_call( data = {} response = await call_mcp_tool( - name=name, - arguments=arguments, user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, **data, # for logging diff --git a/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py new file mode 100644 index 00000000000..c1114d14438 --- /dev/null +++ b/tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server.py @@ -0,0 +1,131 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy._types import UserAPIKeyAuth + + +@pytest.mark.asyncio +async def test_mcp_server_tool_call_body_contains_request_data(): + """Test that proxy_server_request body contains name and arguments""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + mcp_server_tool_call, + set_auth_context, + ) + except ImportError: + pytest.skip("MCP server not available") + + # Setup test data + tool_name = "test_tool" + tool_arguments = {"param1": "value1", "param2": 123} + + # Mock user auth + user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") + set_auth_context(user_api_key_auth) + + # Mock the add_litellm_data_to_request function to capture the data + captured_data = {} + + async def mock_add_litellm_data_to_request( + data, request, user_api_key_dict, proxy_config + ): + captured_data.update(data) + # Simulate the proxy_server_request creation + captured_data["proxy_server_request"] = { + "url": str(request.url), + "method": request.method, + "headers": {}, + "body": data.copy(), # This is what we want to test + } + return captured_data + + # Mock the call_mcp_tool function to avoid actual tool execution + async def mock_call_mcp_tool(*args, **kwargs): + return [{"type": "text", "text": "mocked response"}] + + with patch( + "litellm.proxy.litellm_pre_call_utils.add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ): + with patch( + "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + mock_call_mcp_tool, + ): + with patch( + "litellm.proxy.proxy_server.proxy_config", + MagicMock(), + ): + # Call the function + await mcp_server_tool_call(tool_name, tool_arguments) + + # Verify the body contains the expected data + assert "proxy_server_request" in captured_data + assert "body" in captured_data["proxy_server_request"] + + body = captured_data["proxy_server_request"]["body"] + assert body["name"] == tool_name + assert body["arguments"] == tool_arguments + + +@pytest.mark.asyncio +async def test_mcp_server_tool_call_body_with_none_arguments(): + """Test that proxy_server_request body handles None arguments correctly""" + try: + from litellm.proxy._experimental.mcp_server.server import ( + mcp_server_tool_call, + set_auth_context, + ) + except ImportError: + pytest.skip("MCP server not available") + + # Setup test data + tool_name = "test_tool_no_args" + tool_arguments = None + + # Mock user auth + user_api_key_auth = UserAPIKeyAuth(api_key="test_key", user_id="test_user") + set_auth_context(user_api_key_auth) + + # Mock the add_litellm_data_to_request function to capture the data + captured_data = {} + + async def mock_add_litellm_data_to_request( + data, request, user_api_key_dict, proxy_config + ): + captured_data.update(data) + captured_data["proxy_server_request"] = { + "url": str(request.url), + "method": request.method, + "headers": {}, + "body": data.copy(), + } + return captured_data + + # Mock the call_mcp_tool function + async def mock_call_mcp_tool(*args, **kwargs): + return [{"type": "text", "text": "mocked response"}] + + with patch( + "litellm.proxy.litellm_pre_call_utils.add_litellm_data_to_request", + mock_add_litellm_data_to_request, + ): + with patch( + "litellm.proxy._experimental.mcp_server.server.call_mcp_tool", + mock_call_mcp_tool, + ): + with patch( + "litellm.proxy.proxy_server.proxy_config", + MagicMock(), + ): + # Call the function + await mcp_server_tool_call(tool_name, tool_arguments) + + # Verify the body contains the expected data + assert "proxy_server_request" in captured_data + assert "body" in captured_data["proxy_server_request"] + + body = captured_data["proxy_server_request"]["body"] + assert body["name"] == tool_name + assert body["arguments"] == tool_arguments # Should be None From c01b671539e26c59ac761cbcc881f2e9847f08f3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 19:18:42 -0700 Subject: [PATCH 19/23] fix(new_usage.tsx): fix linting errors --- ui/litellm-dashboard/src/components/new_usage.tsx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index 83b4fb3a7d6..32f65e68774 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -94,7 +94,8 @@ const NewUsagePage: React.FC = ({ cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, - metadata: {} + metadata: {}, + api_key_breakdown: {} }; } modelSpend[model].metrics.spend += metrics.metrics.spend; @@ -140,7 +141,8 @@ const NewUsagePage: React.FC = ({ cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, - metadata: {} + metadata: {}, + api_key_breakdown: {} }; } providerSpend[provider].metrics.spend += metrics.metrics.spend; @@ -185,7 +187,8 @@ const NewUsagePage: React.FC = ({ cache_creation_input_tokens: 0 }, metadata: { - key_alias: metrics.metadata.key_alias + key_alias: metrics.metadata.key_alias, + team_id: null } }; } From 336f072f1ad3d7b7392a4db6bbd5f10377aafbe5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 19:19:32 -0700 Subject: [PATCH 20/23] fix: fix code qa errors --- litellm/proxy/_experimental/mcp_server/server.py | 2 -- litellm/proxy/management_endpoints/mcp_management_endpoints.py | 3 --- 2 files changed, 5 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 93871fe5d01..d07cbb5cb48 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -4,7 +4,6 @@ import asyncio import contextlib -import json from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union from fastapi import FastAPI, HTTPException @@ -12,7 +11,6 @@ from starlette.types import Receive, Scope, Send from litellm._logging import verbose_logger -from litellm.constants import MCP_TOOL_NAME_PREFIX from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, diff --git a/litellm/proxy/management_endpoints/mcp_management_endpoints.py b/litellm/proxy/management_endpoints/mcp_management_endpoints.py index df8dde5188d..3ddd96c1f29 100644 --- a/litellm/proxy/management_endpoints/mcp_management_endpoints.py +++ b/litellm/proxy/management_endpoints/mcp_management_endpoints.py @@ -52,9 +52,6 @@ from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view from litellm.proxy.management_helpers.utils import management_endpoint_wrapper - from litellm.types.proxy.management_endpoints.common_daily_activity import ( - SpendAnalyticsPaginatedResponse, - ) def get_prisma_client_or_throw(message: str): from litellm.proxy.proxy_server import prisma_client From 374acb4682caa5b0182d34b6b87e276a5009308b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 21:31:38 -0700 Subject: [PATCH 21/23] fix(activity_metrics.tsx): fix ui linting errors post-merge --- ui/litellm-dashboard/src/components/activity_metrics.tsx | 5 +++++ ui/litellm-dashboard/src/components/new_usage.tsx | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/ui/litellm-dashboard/src/components/activity_metrics.tsx b/ui/litellm-dashboard/src/components/activity_metrics.tsx index 5eddf98c66d..680c6004195 100644 --- a/ui/litellm-dashboard/src/components/activity_metrics.tsx +++ b/ui/litellm-dashboard/src/components/activity_metrics.tsx @@ -4,6 +4,11 @@ import { AreaChart, BarChart } from '@tremor/react'; import { SpendMetrics, DailyData, ModelActivityData, MetricWithMetadata, KeyMetricWithMetadata, TopApiKeyData } from './usage/types'; import { Collapse } from 'antd'; import { formatNumberWithCommas } from '@/utils/dataUtils'; +import type { CustomTooltipProps } from "@tremor/react"; +import { + valueFormatter, + valueFormatterSpend, +} from "../components/usage/utils/value_formatters"; interface ActivityMetricsProps { modelMetrics: Record; diff --git a/ui/litellm-dashboard/src/components/new_usage.tsx b/ui/litellm-dashboard/src/components/new_usage.tsx index 963c188c425..7a7d437ab17 100644 --- a/ui/litellm-dashboard/src/components/new_usage.tsx +++ b/ui/litellm-dashboard/src/components/new_usage.tsx @@ -199,7 +199,7 @@ const NewUsagePage: React.FC = ({ metrics.metrics.cache_read_input_tokens || 0; providerSpend[provider].metrics.cache_creation_input_tokens += metrics.metrics.cache_creation_input_tokens || 0; - ); + }); }); return Object.entries(providerSpend).map(([provider, metrics]) => ({ From b4e75f9a6f710ce26661e50fb5b50c10072cd29d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 21:33:12 -0700 Subject: [PATCH 22/23] fix(types/utils.py): fix linting error --- litellm/types/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index c859d0a2e81..70b6c97a051 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1821,7 +1821,7 @@ class StandardLoggingUserAPIKeyMetadata(TypedDict): user_api_key_request_route: Optional[str] -class StandardLoggingMCPToolCall(TypedDict, total=False): +class StandardLoggingMCPToolCall(TypedDict, total=False): name: str """ Name of the tool to call @@ -1852,6 +1852,7 @@ class StandardLoggingMCPToolCall(TypedDict, total=False): Namespaced tool name of the MCP tool that the tool call was made to Includes the server name prefix if it exists - eg. `deepwiki-mcp/get_page_content` + """ mcp_server_cost_info: Optional[MCPServerCostInfo] """ From ff4d10e6dff543778a474c01584ff5518cf4b0ed Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 8 Jul 2025 21:43:48 -0700 Subject: [PATCH 23/23] fix(server.py): always have name --- litellm/proxy/_experimental/mcp_server/server.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 10988c69765..6ec68aba1f0 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -129,7 +129,9 @@ async def initialize_session_managers(): await _sse_session_manager_cm.__aenter__() _SESSION_MANAGERS_INITIALIZED = True - verbose_logger.info("MCP Server started with StreamableHTTP and SSE session managers!") + verbose_logger.info( + "MCP Server started with StreamableHTTP and SSE session managers!" + ) async def shutdown_session_managers(): """Shutdown the session managers.""" @@ -228,7 +230,7 @@ async def mcp_server_tool_call( proxy_config=proxy_config, ) else: - data = {} + data = body_data response = await call_mcp_tool( user_api_key_auth=user_api_key_auth, @@ -368,9 +370,13 @@ async def call_mcp_tool( # Try managed server tool first (pass the full prefixed name) # Primary and recommended way to use MCP servers ######################################################### - mcp_server: Optional[MCPServer] = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + mcp_server: Optional[MCPServer] = ( + global_mcp_server_manager._get_mcp_server_from_tool_name(name) + ) if mcp_server: - standard_logging_mcp_tool_call["mcp_server_cost_info"] = (mcp_server.mcp_info or {}).get("mcp_server_cost_info") + standard_logging_mcp_tool_call["mcp_server_cost_info"] = ( + mcp_server.mcp_info or {} + ).get("mcp_server_cost_info") return await _handle_managed_mcp_tool( name=name, # Pass the full name (potentially prefixed) arguments=arguments,