Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions apps/api/src/routes/activity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ const dailyActivitySchema = z.object({
inputCost: z.number(),
outputCost: z.number(),
requestCost: z.number(),
dataStorageCost: z.number(),
errorCount: z.number(),
errorRate: z.number(),
cacheCount: z.number(),
Expand Down Expand Up @@ -155,6 +156,10 @@ activity.openapi(getActivity, async (c) => {
requestCost: sql<number>`COALESCE(SUM(${tables.log.requestCost}), 0)`.as(
"requestCost",
),
dataStorageCost:
sql<number>`COALESCE(SUM(${tables.log.dataStorageCost}), 0)`.as(
"dataStorageCost",
),
errorCount:
sql<number>`SUM(CASE WHEN ${tables.log.hasError} = true THEN 1 ELSE 0 END)`.as(
"errorCount",
Expand Down Expand Up @@ -252,6 +257,7 @@ activity.openapi(getActivity, async (c) => {
const inputCost = Number(day.inputCost);
const outputCost = Number(day.outputCost);
const requestCost = Number(day.requestCost);
const dataStorageCost = Number(day.dataStorageCost);
const errorCount = Number(day.errorCount);
const cacheCount = Number(day.cacheCount);
const discountSavings = Number(day.discountSavings);
Expand All @@ -269,6 +275,7 @@ activity.openapi(getActivity, async (c) => {
inputCost,
outputCost,
requestCost,
dataStorageCost,
errorCount,
errorRate,
cacheCount,
Expand Down
1 change: 1 addition & 0 deletions apps/gateway/src/api.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ describe("test", () => {
name: "Test Organization",
billingEmail: "user",
plan: "pro",
retentionLevel: "retain",
});

await db.insert(tables.userOrganization).values({
Expand Down
43 changes: 42 additions & 1 deletion apps/gateway/src/chat/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { validateSource } from "@/chat/tools/validate-source.js";
import { reportKeyError, reportKeySuccess } from "@/lib/api-key-health.js";
import { calculateCosts } from "@/lib/costs.js";
import { throwIamException, validateModelAccess } from "@/lib/iam.js";
import { insertLog } from "@/lib/logs.js";
import { calculateDataStorageCost, insertLog } from "@/lib/logs.js";

import {
generateCacheKey,
Expand Down Expand Up @@ -1611,6 +1611,17 @@ chat.openapi(completions, async (c) => {
);
}

// Check if organization has credits for data retention costs
// Data storage is billed at $0.01 per 1M tokens, so we need credits when retention is enabled
if (organization && organization.retentionLevel === "retain") {
if (parseFloat(organization.credits || "0") <= 0) {
throw new HTTPException(402, {
message:
"Organization has insufficient credits for data retention. Data retention requires credits for storage costs ($0.01 per 1M tokens). Please add credits or disable data retention in organization settings.",
});
}
}

if (!usedToken) {
throw new HTTPException(500, {
message: `No token`,
Expand Down Expand Up @@ -1822,6 +1833,12 @@ chat.openapi(completions, async (c) => {
estimatedCost: costs.estimatedCost,
discount: costs.discount ?? null,
pricingTier: costs.pricingTier ?? null,
dataStorageCost: calculateDataStorageCost(
promptTokens,
cachedTokens,
completionTokens,
reasoningTokens,
),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
cached: true,
toolResults:
(cachedStreamingResponse.metadata as { toolResults?: any })
Expand Down Expand Up @@ -1930,6 +1947,12 @@ chat.openapi(completions, async (c) => {
estimatedCost: cachedCosts.estimatedCost,
discount: cachedCosts.discount ?? null,
pricingTier: cachedCosts.pricingTier ?? null,
dataStorageCost: calculateDataStorageCost(
cachedResponse.usage?.prompt_tokens,
cachedResponse.usage?.prompt_tokens_details?.cached_tokens,
cachedResponse.usage?.completion_tokens,
cachedResponse.usage?.reasoning_tokens,
),
cached: true,
toolResults: cachedResponse.choices?.[0]?.message?.tool_calls || null,
});
Expand Down Expand Up @@ -2177,6 +2200,7 @@ chat.openapi(completions, async (c) => {
cachedInputCost: null,
requestCost: null,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -2263,6 +2287,7 @@ chat.openapi(completions, async (c) => {
cachedInputCost: null,
requestCost: null,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -2412,6 +2437,7 @@ chat.openapi(completions, async (c) => {
cachedInputCost: null,
requestCost: null,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -3473,6 +3499,12 @@ chat.openapi(completions, async (c) => {
estimatedCost: costs.estimatedCost,
discount: costs.discount,
pricingTier: costs.pricingTier,
dataStorageCost: calculateDataStorageCost(
calculatedPromptTokens,
cachedTokens,
calculatedCompletionTokens,
calculatedReasoningTokens,
),
cached: false,
tools,
toolResults: streamingToolCalls,
Expand Down Expand Up @@ -3643,6 +3675,7 @@ chat.openapi(completions, async (c) => {
requestCost: null,
estimatedCost: false,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -3726,6 +3759,7 @@ chat.openapi(completions, async (c) => {
requestCost: null,
estimatedCost: false,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -3838,6 +3872,7 @@ chat.openapi(completions, async (c) => {
requestCost: null,
estimatedCost: false,
discount: null,
dataStorageCost: "0",
cached: false,
toolResults: null,
});
Expand Down Expand Up @@ -4074,6 +4109,12 @@ chat.openapi(completions, async (c) => {
estimatedCost: costs.estimatedCost,
discount: costs.discount,
pricingTier: costs.pricingTier,
dataStorageCost: calculateDataStorageCost(
calculatedPromptTokens,
cachedTokens,
calculatedCompletionTokens,
calculatedReasoningTokens,
),
cached: false,
tools,
toolResults,
Expand Down
22 changes: 22 additions & 0 deletions apps/gateway/src/lib/logs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,28 @@ export function getUnifiedFinishReason(
return UnifiedFinishReason.UNKNOWN;
}

/**
* Calculate data storage cost based on token usage
* $0.01 per 1M tokens (total tokens = input + cached + output + reasoning)
*/
export function calculateDataStorageCost(
promptTokens: number | string | null | undefined,
cachedTokens: number | string | null | undefined,
completionTokens: number | string | null | undefined,
reasoningTokens: number | string | null | undefined,
): string {
const prompt = Number(promptTokens) || 0;
const cached = Number(cachedTokens) || 0;
const completion = Number(completionTokens) || 0;
const reasoning = Number(reasoningTokens) || 0;

const totalTokens = prompt + cached + completion + reasoning;

// $0.01 per 1M tokens
const cost = (totalTokens / 1_000_000) * 0.01;
return cost.toString();
}

/**
* Insert a log entry into the database.
* This function is extracted to prepare for future implementation using a message queue.
Expand Down
1 change: 1 addition & 0 deletions apps/playground/src/lib/api/v1.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -758,6 +758,7 @@ export interface paths {
inputCost: number;
outputCost: number;
requestCost: number;
dataStorageCost: number;
errorCount: number;
errorRate: number;
cacheCount: number;
Expand Down
17 changes: 12 additions & 5 deletions apps/ui/src/components/dashboard/dashboard-client.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -118,11 +118,10 @@ export function DashboardClient({ initialActivityData }: DashboardClientProps) {
activityData.reduce((sum, day) => sum + day.inputCost, 0) || 0;
const totalOutputCost =
activityData.reduce((sum, day) => sum + day.outputCost, 0) || 0;
const totalDataStorageCost =
activityData.reduce((sum, day) => sum + day.dataStorageCost, 0) || 0;
const totalRequestCost =
activityData.reduce(
(sum, day) => sum + (day.cost - day.inputCost - day.outputCost),
0,
) || 0;
activityData.reduce((sum, day) => sum + day.requestCost, 0) || 0;
const totalSavings =
activityData.reduce((sum, day) => sum + day.discountSavings, 0) || 0;

Expand Down Expand Up @@ -331,7 +330,7 @@ export function DashboardClient({ initialActivityData }: DashboardClientProps) {
<Card>
<CardHeader className="flex flex-row items-center justify-between space-y-0 pb-2">
<CardTitle className="text-sm font-medium">
Cost Estimate
Inference Cost
</CardTitle>
<CircleDollarSign className="text-muted-foreground h-4 w-4" />
</CardHeader>
Expand All @@ -357,6 +356,14 @@ export function DashboardClient({ initialActivityData }: DashboardClientProps) {
</>
)}
</p>
{totalDataStorageCost > 0 && (
<p className="text-muted-foreground text-xs mt-1 pt-1 border-t border-muted">
<span className="text-xs">
LLM Gateway: ${totalDataStorageCost.toFixed(4)}{" "}
storage
</span>
</p>
)}
</>
)}
</CardContent>
Expand Down
77 changes: 58 additions & 19 deletions apps/ui/src/components/dashboard/log-card.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
ChevronUp,
Clock,
Coins,
Info,
Package,
Link as LinkIcon,
Zap,
Expand Down Expand Up @@ -78,18 +79,33 @@ export function LogCard({ log }: { log: Partial<Log> }) {
</div>
<div className="flex-1 space-y-1 min-w-0">
<div className="flex items-start justify-between gap-4">
<p className="font-medium break-words max-w-none line-clamp-2">
{log.content ||
(log.unifiedFinishReason === "tool_calls" && log.toolResults ? (
Array.isArray(log.toolResults) ? (
`Tool calls: ${log.toolResults.map((tr) => tr.function?.name || "unknown").join(", ")}`
) : (
"Tool calls executed"
)
) : (
<i className="italic">–</i>
))}
</p>
<div className="flex items-center gap-2 flex-1 min-w-0">
<p className="font-medium break-words max-w-none line-clamp-2">
{log.content ||
(log.unifiedFinishReason === "tool_calls" && log.toolResults
? Array.isArray(log.toolResults)
? `Tool calls: ${log.toolResults.map((tr) => tr.function?.name || "unknown").join(", ")}`
: "Tool calls executed"
: "---")}
</p>
{!log.content &&
log.unifiedFinishReason !== "tool_calls" &&
!log.hasError && (
<TooltipProvider>
<Tooltip>
<TooltipTrigger asChild>
<Info className="h-4 w-4 text-muted-foreground flex-shrink-0" />
</TooltipTrigger>
<TooltipContent>
<p>
Enable retention in organization policies to store
response content
</p>
</TooltipContent>
</Tooltip>
</TooltipProvider>
)}
</div>
Comment thread
steebchen marked this conversation as resolved.
Resolved
<Badge
variant={
log.hasError
Expand Down Expand Up @@ -380,10 +396,19 @@ export function LogCard({ log }: { log: Partial<Log> }) {
<div>
{log.requestCost ? `$${log.requestCost.toFixed(8)}` : "$0"}
</div>
<div className="text-muted-foreground">Total Cost</div>
<div className="text-muted-foreground">Inference Total</div>
<div className="font-medium">
{log.cost ? `$${log.cost.toFixed(8)}` : "$0"}
</div>
<div className="text-muted-foreground col-span-2 border-t pt-2 mt-1 text-xs">
LLM Gateway Costs
</div>
<div className="text-muted-foreground">Data Storage</div>
<div>
{log.dataStorageCost
? `$${Number(log.dataStorageCost).toFixed(8)}`
: "$0"}
</div>
{log.discount && log.discount !== 1 && (
<>
<div className="text-muted-foreground">
Expand Down Expand Up @@ -680,9 +705,16 @@ export function LogCard({ log }: { log: Partial<Log> }) {
<div className="space-y-2">
<h4 className="text-sm font-medium">Message Context</h4>
<div className="rounded-md border p-3">
<pre className="max-h-60 text-xs overflow-auto whitespace-pre-wrap break-words">
{log.messages ? JSON.stringify(log.messages, null, 2) : "–"}
</pre>
{log.messages ? (
<pre className="max-h-60 text-xs overflow-auto whitespace-pre-wrap break-words">
{JSON.stringify(log.messages, null, 2)}
</pre>
) : (
<p className="text-sm text-muted-foreground italic">
Message data not retained. Enable retention in organization
policies to store request messages.
</p>
)}
</div>
{!!log.responseFormat && (
<div className="mt-3">
Expand Down Expand Up @@ -710,9 +742,16 @@ export function LogCard({ log }: { log: Partial<Log> }) {
<div className="space-y-2">
<h4 className="text-sm font-medium">Response</h4>
<div className="rounded-md border p-3">
<pre className="max-h-60 text-xs overflow-auto whitespace-pre-wrap break-words">
{log.content || "–"}
</pre>
{log.content ? (
<pre className="max-h-60 text-xs overflow-auto whitespace-pre-wrap break-words">
{log.content}
</pre>
) : (
<p className="text-sm text-muted-foreground italic">
Response content not retained. Enable retention in
organization policies to store response data.
</p>
)}
</div>
</div>
</div>
Expand Down
Loading