diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index 89ed9bf7413..c1a9034e8a1 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -1988,6 +1988,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, } if bifrostUsage != nil { response.Usage = bifrostUsage + response.Speed = chunk.Usage.Speed } // Carry the sandbox container on the message_delta event so the reverse // converter can re-emit it (Anthropic delivers it here, not earlier). @@ -7816,4 +7817,4 @@ func generateSyntheticInputJSONDeltas(argumentsJSON string, contentIndex *int) [ } return events -} \ No newline at end of file +} diff --git a/core/providers/anthropic/utils_test.go b/core/providers/anthropic/utils_test.go index 9f0f3e4d65c..d3307a0c158 100644 --- a/core/providers/anthropic/utils_test.go +++ b/core/providers/anthropic/utils_test.go @@ -3475,3 +3475,46 @@ func TestStripEmptyThinkingBlocks(t *testing.T) { }) } } + +// TestFastMode_StreamingForwardsSpeed verifies the streaming Responses converter +// forwards the served speed onto the response, so streamed fast-mode requests +// bill at fast rates (parity with the non-streaming ToBifrostResponsesResponse +// path). Without this, tier.isFast is false for streams and cache/input/output +// all bill at standard rates. +func TestFastMode_StreamingForwardsSpeed(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyIntegrationType, "anthropic") + state := AcquireAnthropicResponsesStreamState() + defer ReleaseAnthropicResponsesStreamState(state) + + // Final usage arrives on message_delta: speed:"fast" + 5m cache-creation tokens. + raw := `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":2,"output_tokens":135,"cache_creation_input_tokens":44667,"cache_creation":{"ephemeral_5m_input_tokens":44667,"ephemeral_1h_input_tokens":0},"speed":"fast"}}` + var chunk AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(raw), &chunk); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + + responses, bErr, _ := chunk.ToBifrostResponsesStream(ctx, 0, state) + if bErr != nil { + t.Fatalf("ToBifrostResponsesStream error: %v", bErr) + } + + var sawUsage bool + for _, r := range responses { + if r.Response == nil || r.Response.Usage == nil { + continue + } + sawUsage = true + if r.Response.Speed == nil || *r.Response.Speed != "fast" { + t.Fatalf("streamed message_delta did not forward speed=fast; got %v", r.Response.Speed) + } + // Cache-creation tokens must survive so the fast cache rate applies. + if r.Response.Usage.InputTokensDetails == nil || + r.Response.Usage.InputTokensDetails.CachedWriteTokens != 44667 { + t.Fatalf("cache-creation tokens not carried onto streamed usage") + } + } + if !sawUsage { + t.Fatalf("no usage-bearing response emitted from message_delta") + } +} diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 561b6bdbf33..84a0fcb36fa 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -436,6 +436,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_model_pricing_is_deprecated_column"}, run: migrationAddModelPricingIsDeprecatedColumn}, {IDs: []string{"add_mcp_client_tool_execution_timeout_column"}, run: migrationAddMCPClientToolExecutionTimeoutColumn}, {IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn}, + {IDs: []string{"add_fast_mode_cache_pricing_columns"}, run: migrationAddFastModeCachePricingColumns}, } // quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes. @@ -7797,6 +7798,45 @@ func migrationAddFastModePricingColumns(ctx context.Context, db *gorm.DB, logger return nil } +// migrationAddFastModeCachePricingColumns adds fast-mode cache pricing columns +// for Anthropic (speed:"fast"). Caching multipliers stack on the fast base input +// rate, so cache tokens need dedicated fast rates instead of the standard ones. +func migrationAddFastModeCachePricingColumns(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_fast_mode_cache_pricing_columns" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + columns := []string{ + "cache_creation_input_token_cost_fast", + "cache_creation_input_token_cost_above_1hr_fast", + "cache_read_input_token_cost_fast", + } + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := addColumnIfNotExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to add column %s: %w", field, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := dropColumnIfExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to drop column %s: %w", field, err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while running fast mode cache pricing columns migration: %s", err.Error()) + } + return nil +} + // migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { migrationName := "add_whitelisted_routes_json_column" diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index 524cf1b2cb5..e4aeedc961c 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -2491,6 +2491,9 @@ var pricingSyncUpdateColumns = []string{ "cache_read_input_image_token_cost", "cache_read_input_token_cost_above_272k_tokens", "cache_read_input_token_cost_above_272k_tokens_priority", + "cache_creation_input_token_cost_fast", + "cache_creation_input_token_cost_above_1hr_fast", + "cache_read_input_token_cost_fast", // Costs - Image "input_cost_per_image", "input_cost_per_pixel", diff --git a/framework/configstore/tables/modelpricing.go b/framework/configstore/tables/modelpricing.go index d1c7385dd7b..34d157e80b8 100644 --- a/framework/configstore/tables/modelpricing.go +++ b/framework/configstore/tables/modelpricing.go @@ -30,7 +30,7 @@ type TableModelPricing struct { InputCostPerTokenFlex *float64 `gorm:"default:null;column:input_cost_per_token_flex" json:"input_cost_per_token_flex,omitempty"` OutputCostPerTokenFlex *float64 `gorm:"default:null;column:output_cost_per_token_flex" json:"output_cost_per_token_flex,omitempty"` // Fast mode (Anthropic research preview, speed:"fast" on Opus 4.6/4.7/4.8). - // Flat rate across the full context window; cache tokens bill at standard cache rates. + // Flat rate across the full context window; cache tokens use the _fast cache columns below. InputCostPerTokenFast *float64 `gorm:"default:null;column:input_cost_per_token_fast" json:"input_cost_per_token_fast,omitempty"` OutputCostPerTokenFast *float64 `gorm:"default:null;column:output_cost_per_token_fast" json:"output_cost_per_token_fast,omitempty"` InputCostPerCharacter *float64 `gorm:"default:null;column:input_cost_per_character" json:"input_cost_per_character,omitempty"` @@ -65,6 +65,10 @@ type TableModelPricing struct { CacheReadInputImageTokenCost *float64 `gorm:"default:null;column:cache_read_input_image_token_cost" json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens" json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens_priority" json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. + CacheCreationInputTokenCostFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_fast" json:"cache_creation_input_token_cost_fast,omitempty"` + CacheCreationInputTokenCostAbove1hrFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_above_1hr_fast" json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` + CacheReadInputTokenCostFast *float64 `gorm:"default:null;column:cache_read_input_token_cost_fast" json:"cache_read_input_token_cost_fast,omitempty"` // Costs - Image InputCostPerImage *float64 `gorm:"default:null;column:input_cost_per_image" json:"input_cost_per_image,omitempty"` diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index c7a89b2388d..8e36f108d6c 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -937,6 +937,10 @@ func tieredAudioTokenOutputRate(pricing *configstoreTables.TableModelPricing, to } func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheReadInputTokenCostFast != nil { + return *pricing.CacheReadInputTokenCostFast + } if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil { return *pricing.CacheReadInputTokenCostFlex } @@ -969,6 +973,10 @@ func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, // OpenAI's pricing model (the only provider that uses flex tier). Only cache read // has a flex-specific rate. func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheCreationInputTokenCostFast != nil { + return *pricing.CacheCreationInputTokenCostFast + } if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil { return *pricing.CacheCreationInputTokenCostAbove200kTokens } @@ -979,6 +987,10 @@ func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPric } func tieredCacheCreationInputAbove1hrTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheCreationInputTokenCostAbove1hrFast != nil { + return *pricing.CacheCreationInputTokenCostAbove1hrFast + } if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens != nil { return *pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens } diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 7c40b590d06..3ddf9befe83 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -205,14 +205,16 @@ func TestComputeTextCost_FastMode_FallsBackWhenUnconfigured(t *testing.T) { assert.InDelta(t, 1000*0.000005+500*0.000025, fast, 1e-12) } -func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) { - // Per design: cache tokens on a fast request bill at standard cache rates; - // only the non-cached input and the output use the fast rate. +func TestComputeTextCost_FastMode_UsesFastCacheRates(t *testing.T) { + // Fast mode has dedicated cache columns; when set, cache tokens bill at the + // _fast rate, not the standard rate. p := chatPricing(0.000005, 0.000025) p.InputCostPerTokenFast = bifrost.Ptr(0.00001) p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) - p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read - p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read (ignored in fast) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast) + p.CacheReadInputTokenCostFast = bifrost.Ptr(0.000001) // fast read + p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write usage := &schemas.BifrostLLMUsage{ PromptTokens: 2000, @@ -225,12 +227,66 @@ func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) { } fast := computeTextCost(&p, usage, serviceTier{isFast: true}) - // Input: non-cached (2000-1500-200)*fast + read 1500*stdRead + write 200*stdWrite - // = 300*0.00001 + 1500*0.0000005 + 200*0.00000625 = 0.003 + 0.00075 + 0.00125 = 0.0019(? recompute) + // non-cached 300*fast + read 1500*fastRead + write 200*fastWrite + output 500*fast + expected := 300*0.00001 + 1500*0.000001 + 200*0.0000125 + 500*0.00005 + assert.InDelta(t, expected, fast, 1e-12) +} + +func TestComputeTextCost_FastMode_CacheFallsBackToStandardWhenFastUnset(t *testing.T) { + // When the _fast cache columns are absent, cache tokens fall back to standard + // cache rates (mirrors the input/output fast fallback) — the flag is a no-op. + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 2000, + CompletionTokens: 500, + TotalTokens: 2500, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 1500, + CachedWriteTokens: 200, + }, + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // input/output use fast; cache uses standard. expected := 300*0.00001 + 1500*0.0000005 + 200*0.00000625 + 500*0.00005 assert.InDelta(t, expected, fast, 1e-12) } +// TestComputeTextCost_FastMode_Opus48CacheRegression pins the reported real-world +// miscalculation: fast mode + cache_control billed cache creation at the standard +// rate. Opus 4.8: fast $10/$50, fast 5m cache write $12.50 per MTok. +func TestComputeTextCost_FastMode_Opus48CacheRegression(t *testing.T) { + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast) + p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write + + // input_tokens=2, cache_creation=44667 (all 5m), output=135. PromptTokens + // carries the cache-creation tokens (Anthropic responses usage mapping). + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 44669, + CompletionTokens: 135, + TotalTokens: 44804, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedWriteTokens: 44667, + CachedWriteTokenDetails: &schemas.ChatCachedWriteTokenDetails{ + CachedWriteTokens5m: 44667, + }, + }, + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // 2*$10/M + 44667*$12.50/M (fast 5m cache) + 135*$50/M = $0.565108 + expected := 2*0.00001 + 44667*0.0000125 + 135*0.00005 + assert.InDelta(t, expected, fast, 1e-9) +} + func TestTierFromResponse_Speed(t *testing.T) { assert.False(t, tierFromResponse(nil, nil).isFast) assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard")).isFast) diff --git a/framework/modelcatalog/datasheet/overrides.go b/framework/modelcatalog/datasheet/overrides.go index 67c335e4b7e..84d19c38ea7 100644 --- a/framework/modelcatalog/datasheet/overrides.go +++ b/framework/modelcatalog/datasheet/overrides.go @@ -289,6 +289,9 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options) {dst: &patched.CacheReadInputTokenCostAbove200kTokensPriority, src: override.CacheReadInputTokenCostAbove200kTokensPriority}, {dst: &patched.CacheReadInputTokenCostAbove272kTokens, src: override.CacheReadInputTokenCostAbove272kTokens}, {dst: &patched.CacheReadInputTokenCostAbove272kTokensPriority, src: override.CacheReadInputTokenCostAbove272kTokensPriority}, + {dst: &patched.CacheCreationInputTokenCostFast, src: override.CacheCreationInputTokenCostFast}, + {dst: &patched.CacheCreationInputTokenCostAbove1hrFast, src: override.CacheCreationInputTokenCostAbove1hrFast}, + {dst: &patched.CacheReadInputTokenCostFast, src: override.CacheReadInputTokenCostFast}, {dst: &patched.InputCostPerTokenBatches, src: override.InputCostPerTokenBatches}, {dst: &patched.OutputCostPerTokenBatches, src: override.OutputCostPerTokenBatches}, {dst: &patched.InputCostPerImageToken, src: override.InputCostPerImageToken}, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 2616bf8694f..28182156971 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -139,6 +139,10 @@ type Options struct { CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. + CacheCreationInputTokenCostFast *float64 `json:"cache_creation_input_token_cost_fast,omitempty"` + CacheCreationInputTokenCostAbove1hrFast *float64 `json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` + CacheReadInputTokenCostFast *float64 `json:"cache_read_input_token_cost_fast,omitempty"` // Costs - Image InputCostPerImage *float64 `json:"input_cost_per_image,omitempty"` @@ -593,6 +597,9 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables. CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority, + CacheCreationInputTokenCostFast: entry.CacheCreationInputTokenCostFast, + CacheCreationInputTokenCostAbove1hrFast: entry.CacheCreationInputTokenCostAbove1hrFast, + CacheReadInputTokenCostFast: entry.CacheReadInputTokenCostFast, InputCostPerImage: entry.InputCostPerImage, InputCostPerPixel: entry.InputCostPerPixel, @@ -670,6 +677,9 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority, + CacheCreationInputTokenCostFast: pricing.CacheCreationInputTokenCostFast, + CacheCreationInputTokenCostAbove1hrFast: pricing.CacheCreationInputTokenCostAbove1hrFast, + CacheReadInputTokenCostFast: pricing.CacheReadInputTokenCostFast, InputCostPerImage: pricing.InputCostPerImage, InputCostPerPixel: pricing.InputCostPerPixel, diff --git a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx index 4e4fb5ee703..7c3363d97af 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx +++ b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx @@ -170,6 +170,9 @@ export const PRICING_FIELDS = [ }, { key: "cache_read_input_token_cost_priority", label: "Cache read / token (priority)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_flex", label: "Cache read / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_creation_input_token_cost_fast", label: "Cache creation / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_creation_input_token_cost_above_1hr_fast", label: "Cache creation / token (>1hr, fast)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_read_input_token_cost_fast", label: "Cache read / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_above_200k_tokens_priority", label: "Cache read / token (>200k, priority)", diff --git a/ui/lib/types/governance.ts b/ui/lib/types/governance.ts index 58cdeb83c59..3921cbb4a0e 100644 --- a/ui/lib/types/governance.ts +++ b/ui/lib/types/governance.ts @@ -455,6 +455,9 @@ export interface PricingOverridePatch { cache_read_input_image_token_cost?: number; cache_read_input_token_cost_above_272k_tokens?: number; cache_read_input_token_cost_above_272k_tokens_priority?: number; + cache_creation_input_token_cost_fast?: number; + cache_creation_input_token_cost_above_1hr_fast?: number; + cache_read_input_token_cost_fast?: number; // Image input_cost_per_image_token?: number; output_cost_per_image_token?: number;