Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion core/providers/anthropic/responses.go
Original file line number Diff line number Diff line change
Expand Up @@ -1988,6 +1988,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context,
}
if bifrostUsage != nil {
response.Usage = bifrostUsage
response.Speed = chunk.Usage.Speed
Comment thread
TejasGhatte marked this conversation as resolved.
Resolved
}
// Carry the sandbox container on the message_delta event so the reverse
// converter can re-emit it (Anthropic delivers it here, not earlier).
Expand Down Expand Up @@ -7816,4 +7817,4 @@ func generateSyntheticInputJSONDeltas(argumentsJSON string, contentIndex *int) [
}

return events
}
}
43 changes: 43 additions & 0 deletions core/providers/anthropic/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3475,3 +3475,46 @@ func TestStripEmptyThinkingBlocks(t *testing.T) {
})
}
}

// TestFastMode_StreamingForwardsSpeed verifies the streaming Responses converter
// forwards the served speed onto the response, so streamed fast-mode requests
// bill at fast rates (parity with the non-streaming ToBifrostResponsesResponse
// path). Without this, tier.isFast is false for streams and cache/input/output
// all bill at standard rates.
func TestFastMode_StreamingForwardsSpeed(t *testing.T) {
ctx := schemas.NewBifrostContext(nil, time.Time{})
ctx.SetValue(schemas.BifrostContextKeyIntegrationType, "anthropic")
state := AcquireAnthropicResponsesStreamState()
defer ReleaseAnthropicResponsesStreamState(state)

// Final usage arrives on message_delta: speed:"fast" + 5m cache-creation tokens.
raw := `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":2,"output_tokens":135,"cache_creation_input_tokens":44667,"cache_creation":{"ephemeral_5m_input_tokens":44667,"ephemeral_1h_input_tokens":0},"speed":"fast"}}`
var chunk AnthropicStreamEvent
if err := sonic.Unmarshal([]byte(raw), &chunk); err != nil {
t.Fatalf("unmarshal event: %v", err)
}

responses, bErr, _ := chunk.ToBifrostResponsesStream(ctx, 0, state)
if bErr != nil {
t.Fatalf("ToBifrostResponsesStream error: %v", bErr)
}

var sawUsage bool
for _, r := range responses {
if r.Response == nil || r.Response.Usage == nil {
continue
}
sawUsage = true
if r.Response.Speed == nil || *r.Response.Speed != "fast" {
t.Fatalf("streamed message_delta did not forward speed=fast; got %v", r.Response.Speed)
}
// Cache-creation tokens must survive so the fast cache rate applies.
if r.Response.Usage.InputTokensDetails == nil ||
r.Response.Usage.InputTokensDetails.CachedWriteTokens != 44667 {
t.Fatalf("cache-creation tokens not carried onto streamed usage")
}
}
if !sawUsage {
t.Fatalf("no usage-bearing response emitted from message_delta")
}
}
40 changes: 40 additions & 0 deletions framework/configstore/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ var configstoreMigrationSteps = []migrationStep{
{IDs: []string{"add_model_pricing_is_deprecated_column"}, run: migrationAddModelPricingIsDeprecatedColumn},
{IDs: []string{"add_mcp_client_tool_execution_timeout_column"}, run: migrationAddMCPClientToolExecutionTimeoutColumn},
{IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn},
{IDs: []string{"add_fast_mode_cache_pricing_columns"}, run: migrationAddFastModeCachePricingColumns},
}

// quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes.
Expand Down Expand Up @@ -7797,6 +7798,45 @@ func migrationAddFastModePricingColumns(ctx context.Context, db *gorm.DB, logger
return nil
}

// migrationAddFastModeCachePricingColumns adds fast-mode cache pricing columns
// for Anthropic (speed:"fast"). Caching multipliers stack on the fast base input
// rate, so cache tokens need dedicated fast rates instead of the standard ones.
func migrationAddFastModeCachePricingColumns(ctx context.Context, db *gorm.DB, logger schemas.Logger) error {
migrationName := "add_fast_mode_cache_pricing_columns"
logger.Info("[configstore] starting migration %s", migrationName)
defer logger.Info("[configstore] finished migration %s", migrationName)
columns := []string{
"cache_creation_input_token_cost_fast",
"cache_creation_input_token_cost_above_1hr_fast",
"cache_read_input_token_cost_fast",
}
m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{
ID: migrationName,
Migrate: func(tx *gorm.DB) error {
tx = tx.WithContext(ctx)
for _, field := range columns {
if err := addColumnIfNotExists(tx, logger, &tables.TableModelPricing{}, field); err != nil {
return fmt.Errorf("failed to add column %s: %w", field, err)
}
}
return nil
},
Rollback: func(tx *gorm.DB) error {
tx = tx.WithContext(ctx)
for _, field := range columns {
if err := dropColumnIfExists(tx, logger, &tables.TableModelPricing{}, field); err != nil {
return fmt.Errorf("failed to drop column %s: %w", field, err)
}
}
return nil
},
}})
if err := m.Migrate(); err != nil {
return fmt.Errorf("error while running fast mode cache pricing columns migration: %s", err.Error())
}
return nil
}

// migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table
func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error {
migrationName := "add_whitelisted_routes_json_column"
Expand Down
3 changes: 3 additions & 0 deletions framework/configstore/rdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -2491,6 +2491,9 @@ var pricingSyncUpdateColumns = []string{
"cache_read_input_image_token_cost",
"cache_read_input_token_cost_above_272k_tokens",
"cache_read_input_token_cost_above_272k_tokens_priority",
"cache_creation_input_token_cost_fast",
"cache_creation_input_token_cost_above_1hr_fast",
"cache_read_input_token_cost_fast",
// Costs - Image
"input_cost_per_image",
"input_cost_per_pixel",
Expand Down
6 changes: 5 additions & 1 deletion framework/configstore/tables/modelpricing.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ type TableModelPricing struct {
InputCostPerTokenFlex *float64 `gorm:"default:null;column:input_cost_per_token_flex" json:"input_cost_per_token_flex,omitempty"`
OutputCostPerTokenFlex *float64 `gorm:"default:null;column:output_cost_per_token_flex" json:"output_cost_per_token_flex,omitempty"`
// Fast mode (Anthropic research preview, speed:"fast" on Opus 4.6/4.7/4.8).
// Flat rate across the full context window; cache tokens bill at standard cache rates.
// Flat rate across the full context window; cache tokens use the _fast cache columns below.
InputCostPerTokenFast *float64 `gorm:"default:null;column:input_cost_per_token_fast" json:"input_cost_per_token_fast,omitempty"`
OutputCostPerTokenFast *float64 `gorm:"default:null;column:output_cost_per_token_fast" json:"output_cost_per_token_fast,omitempty"`
InputCostPerCharacter *float64 `gorm:"default:null;column:input_cost_per_character" json:"input_cost_per_character,omitempty"`
Expand Down Expand Up @@ -65,6 +65,10 @@ type TableModelPricing struct {
CacheReadInputImageTokenCost *float64 `gorm:"default:null;column:cache_read_input_image_token_cost" json:"cache_read_input_image_token_cost,omitempty"`
CacheReadInputTokenCostAbove272kTokens *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens" json:"cache_read_input_token_cost_above_272k_tokens,omitempty"`
CacheReadInputTokenCostAbove272kTokensPriority *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens_priority" json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"`
// Fast mode (Anthropic) cache rates — flat across the full context window, no tiering.
CacheCreationInputTokenCostFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_fast" json:"cache_creation_input_token_cost_fast,omitempty"`
CacheCreationInputTokenCostAbove1hrFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_above_1hr_fast" json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"`
CacheReadInputTokenCostFast *float64 `gorm:"default:null;column:cache_read_input_token_cost_fast" json:"cache_read_input_token_cost_fast,omitempty"`

// Costs - Image
InputCostPerImage *float64 `gorm:"default:null;column:input_cost_per_image" json:"input_cost_per_image,omitempty"`
Expand Down
12 changes: 12 additions & 0 deletions framework/modelcatalog/datasheet/cost.go
Original file line number Diff line number Diff line change
Expand Up @@ -937,6 +937,10 @@ func tieredAudioTokenOutputRate(pricing *configstoreTables.TableModelPricing, to
}

func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 {
// Fast mode (Anthropic) is a flat rate across the full context window.
if tier.isFast && pricing.CacheReadInputTokenCostFast != nil {
return *pricing.CacheReadInputTokenCostFast
}
if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil {
return *pricing.CacheReadInputTokenCostFlex
}
Expand Down Expand Up @@ -969,6 +973,10 @@ func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing,
// OpenAI's pricing model (the only provider that uses flex tier). Only cache read
// has a flex-specific rate.
func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 {
// Fast mode (Anthropic) is a flat rate across the full context window.
if tier.isFast && pricing.CacheCreationInputTokenCostFast != nil {
return *pricing.CacheCreationInputTokenCostFast
}
if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil {
return *pricing.CacheCreationInputTokenCostAbove200kTokens
}
Expand All @@ -979,6 +987,10 @@ func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPric
}

func tieredCacheCreationInputAbove1hrTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 {
// Fast mode (Anthropic) is a flat rate across the full context window.
if tier.isFast && pricing.CacheCreationInputTokenCostAbove1hrFast != nil {
return *pricing.CacheCreationInputTokenCostAbove1hrFast
}
if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens != nil {
return *pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens
}
Expand Down
70 changes: 63 additions & 7 deletions framework/modelcatalog/datasheet/cost_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,14 +205,16 @@ func TestComputeTextCost_FastMode_FallsBackWhenUnconfigured(t *testing.T) {
assert.InDelta(t, 1000*0.000005+500*0.000025, fast, 1e-12)
}

func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) {
// Per design: cache tokens on a fast request bill at standard cache rates;
// only the non-cached input and the output use the fast rate.
func TestComputeTextCost_FastMode_UsesFastCacheRates(t *testing.T) {
// Fast mode has dedicated cache columns; when set, cache tokens bill at the
// _fast rate, not the standard rate.
p := chatPricing(0.000005, 0.000025)
p.InputCostPerTokenFast = bifrost.Ptr(0.00001)
p.OutputCostPerTokenFast = bifrost.Ptr(0.00005)
p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read
p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write
p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read (ignored in fast)
p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast)
p.CacheReadInputTokenCostFast = bifrost.Ptr(0.000001) // fast read
p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write

usage := &schemas.BifrostLLMUsage{
PromptTokens: 2000,
Expand All @@ -225,12 +227,66 @@ func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) {
}

fast := computeTextCost(&p, usage, serviceTier{isFast: true})
// Input: non-cached (2000-1500-200)*fast + read 1500*stdRead + write 200*stdWrite
// = 300*0.00001 + 1500*0.0000005 + 200*0.00000625 = 0.003 + 0.00075 + 0.00125 = 0.0019(? recompute)
// non-cached 300*fast + read 1500*fastRead + write 200*fastWrite + output 500*fast
expected := 300*0.00001 + 1500*0.000001 + 200*0.0000125 + 500*0.00005
assert.InDelta(t, expected, fast, 1e-12)
}

func TestComputeTextCost_FastMode_CacheFallsBackToStandardWhenFastUnset(t *testing.T) {
// When the _fast cache columns are absent, cache tokens fall back to standard
// cache rates (mirrors the input/output fast fallback) — the flag is a no-op.
p := chatPricing(0.000005, 0.000025)
p.InputCostPerTokenFast = bifrost.Ptr(0.00001)
p.OutputCostPerTokenFast = bifrost.Ptr(0.00005)
p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005)
p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625)

usage := &schemas.BifrostLLMUsage{
PromptTokens: 2000,
CompletionTokens: 500,
TotalTokens: 2500,
PromptTokensDetails: &schemas.ChatPromptTokensDetails{
CachedReadTokens: 1500,
CachedWriteTokens: 200,
},
}

fast := computeTextCost(&p, usage, serviceTier{isFast: true})
// input/output use fast; cache uses standard.
expected := 300*0.00001 + 1500*0.0000005 + 200*0.00000625 + 500*0.00005
assert.InDelta(t, expected, fast, 1e-12)
}

// TestComputeTextCost_FastMode_Opus48CacheRegression pins the reported real-world
// miscalculation: fast mode + cache_control billed cache creation at the standard
// rate. Opus 4.8: fast $10/$50, fast 5m cache write $12.50 per MTok.
func TestComputeTextCost_FastMode_Opus48CacheRegression(t *testing.T) {
p := chatPricing(0.000005, 0.000025)
p.InputCostPerTokenFast = bifrost.Ptr(0.00001)
p.OutputCostPerTokenFast = bifrost.Ptr(0.00005)
p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast)
p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write

// input_tokens=2, cache_creation=44667 (all 5m), output=135. PromptTokens
// carries the cache-creation tokens (Anthropic responses usage mapping).
usage := &schemas.BifrostLLMUsage{
PromptTokens: 44669,
CompletionTokens: 135,
TotalTokens: 44804,
PromptTokensDetails: &schemas.ChatPromptTokensDetails{
CachedWriteTokens: 44667,
CachedWriteTokenDetails: &schemas.ChatCachedWriteTokenDetails{
CachedWriteTokens5m: 44667,
},
},
}

fast := computeTextCost(&p, usage, serviceTier{isFast: true})
// 2*$10/M + 44667*$12.50/M (fast 5m cache) + 135*$50/M = $0.565108
expected := 2*0.00001 + 44667*0.0000125 + 135*0.00005
assert.InDelta(t, expected, fast, 1e-9)
}

func TestTierFromResponse_Speed(t *testing.T) {
assert.False(t, tierFromResponse(nil, nil).isFast)
assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard")).isFast)
Expand Down
3 changes: 3 additions & 0 deletions framework/modelcatalog/datasheet/overrides.go
Original file line number Diff line number Diff line change
Expand Up @@ -289,6 +289,9 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options)
{dst: &patched.CacheReadInputTokenCostAbove200kTokensPriority, src: override.CacheReadInputTokenCostAbove200kTokensPriority},
{dst: &patched.CacheReadInputTokenCostAbove272kTokens, src: override.CacheReadInputTokenCostAbove272kTokens},
{dst: &patched.CacheReadInputTokenCostAbove272kTokensPriority, src: override.CacheReadInputTokenCostAbove272kTokensPriority},
{dst: &patched.CacheCreationInputTokenCostFast, src: override.CacheCreationInputTokenCostFast},
{dst: &patched.CacheCreationInputTokenCostAbove1hrFast, src: override.CacheCreationInputTokenCostAbove1hrFast},
{dst: &patched.CacheReadInputTokenCostFast, src: override.CacheReadInputTokenCostFast},
{dst: &patched.InputCostPerTokenBatches, src: override.InputCostPerTokenBatches},
{dst: &patched.OutputCostPerTokenBatches, src: override.OutputCostPerTokenBatches},
{dst: &patched.InputCostPerImageToken, src: override.InputCostPerImageToken},
Expand Down
10 changes: 10 additions & 0 deletions framework/modelcatalog/datasheet/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,10 @@ type Options struct {
CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"`
CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"`
CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"`
// Fast mode (Anthropic) cache rates — flat across the full context window, no tiering.
CacheCreationInputTokenCostFast *float64 `json:"cache_creation_input_token_cost_fast,omitempty"`
CacheCreationInputTokenCostAbove1hrFast *float64 `json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"`
CacheReadInputTokenCostFast *float64 `json:"cache_read_input_token_cost_fast,omitempty"`

// Costs - Image
InputCostPerImage *float64 `json:"input_cost_per_image,omitempty"`
Expand Down Expand Up @@ -593,6 +597,9 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables.
CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost,
CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens,
CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority,
CacheCreationInputTokenCostFast: entry.CacheCreationInputTokenCostFast,
CacheCreationInputTokenCostAbove1hrFast: entry.CacheCreationInputTokenCostAbove1hrFast,
CacheReadInputTokenCostFast: entry.CacheReadInputTokenCostFast,

InputCostPerImage: entry.InputCostPerImage,
InputCostPerPixel: entry.InputCostPerPixel,
Expand Down Expand Up @@ -670,6 +677,9 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E
CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost,
CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens,
CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority,
CacheCreationInputTokenCostFast: pricing.CacheCreationInputTokenCostFast,
CacheCreationInputTokenCostAbove1hrFast: pricing.CacheCreationInputTokenCostAbove1hrFast,
CacheReadInputTokenCostFast: pricing.CacheReadInputTokenCostFast,

InputCostPerImage: pricing.InputCostPerImage,
InputCostPerPixel: pricing.InputCostPerPixel,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,9 @@ export const PRICING_FIELDS = [
},
{ key: "cache_read_input_token_cost_priority", label: "Cache read / token (priority)", group: "chat", requestTypeGroups: ["chat"] },
{ key: "cache_read_input_token_cost_flex", label: "Cache read / token (flex)", group: "chat", requestTypeGroups: ["chat"] },
{ key: "cache_creation_input_token_cost_fast", label: "Cache creation / token (fast)", group: "chat", requestTypeGroups: ["chat"] },
{ key: "cache_creation_input_token_cost_above_1hr_fast", label: "Cache creation / token (>1hr, fast)", group: "chat", requestTypeGroups: ["chat"] },
{ key: "cache_read_input_token_cost_fast", label: "Cache read / token (fast)", group: "chat", requestTypeGroups: ["chat"] },
{
key: "cache_read_input_token_cost_above_200k_tokens_priority",
label: "Cache read / token (>200k, priority)",
Expand Down
3 changes: 3 additions & 0 deletions ui/lib/types/governance.ts
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,9 @@ export interface PricingOverridePatch {
cache_read_input_image_token_cost?: number;
cache_read_input_token_cost_above_272k_tokens?: number;
cache_read_input_token_cost_above_272k_tokens_priority?: number;
cache_creation_input_token_cost_fast?: number;
cache_creation_input_token_cost_above_1hr_fast?: number;
cache_read_input_token_cost_fast?: number;
// Image
input_cost_per_image_token?: number;
output_cost_per_image_token?: number;
Expand Down
Loading