From 714a1253d2d24ef407eccfab8b7fdc3f0120fc1e Mon Sep 17 00:00:00 2001 From: Tejas Ghatte <64637256+TejasGhatte@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:40:54 +0530 Subject: [PATCH 1/5] fix: annthropic cache rate for fast mode (#5063) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes a cost calculation bug where Anthropic prompt-cache tokens (both cache read and cache creation) were billed at standard rates even when the request used fast mode. Per Anthropic's pricing model, cache multipliers stack on top of the fast base input rate, meaning cache token costs should scale by the fast/standard input ratio rather than remaining at the standard cache rate. - Introduced `fastCacheRate`, a helper that scales a standard cache rate to the fast-mode equivalent by multiplying it by the fast/standard input price ratio. - Applied `fastCacheRate` as an early-return guard in `tieredCacheReadInputTokenRate`, `tieredCacheCreationInputTokenRate`, and `tieredCacheCreationInputAbove1hrTokenRate` so fast-mode requests use the correctly scaled cache rates. - Updated the existing fast-mode cache test (`TestComputeTextCost_FastMode_CacheStacksOnFastBase`) to reflect the correct expected values (cache tokens now scale 2× when the fast input rate is 2× the standard rate). - Added a regression test (`TestComputeTextCost_FastMode_Opus48CacheRegression`) pinning the real-world miscalculation observed with Anthropic Opus 4.8 fast mode + `cache_control`, where 44,667 5-minute cache-creation tokens were billed at the standard $6.25/MTok instead of the fast-scaled $12.50/MTok. - [x] Bug fix - [x] Core (Go) ```sh go test ./framework/modelcatalog/datasheet/... ``` The new regression test `TestComputeTextCost_FastMode_Opus48CacheRegression` will fail on the unfixed code and pass after this change. The updated `TestComputeTextCost_FastMode_CacheStacksOnFastBase` validates the general scaling behaviour. - [x] No None. - [ ] I read `docs/contributing/README.md` and followed the guidelines - [x] I added/updated tests where appropriate - [ ] I updated documentation where needed - [x] I verified builds succeed (Go and UI) - [ ] I verified the CI pipeline passes locally if applicable --- core/providers/anthropic/responses.go | 3 +- core/providers/anthropic/utils_test.go | 43 ++++++++++++ framework/configstore/migrations.go | 40 +++++++++++ framework/configstore/rdb.go | 3 + framework/configstore/tables/modelpricing.go | 6 +- framework/modelcatalog/datasheet/cost.go | 12 ++++ framework/modelcatalog/datasheet/cost_test.go | 70 +++++++++++++++++-- framework/modelcatalog/datasheet/overrides.go | 3 + framework/modelcatalog/datasheet/types.go | 10 +++ .../overrides/pricingOverrideSheet.tsx | 3 + ui/lib/types/governance.ts | 3 + 11 files changed, 187 insertions(+), 9 deletions(-) diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index 89ed9bf7413..c1a9034e8a1 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -1988,6 +1988,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, } if bifrostUsage != nil { response.Usage = bifrostUsage + response.Speed = chunk.Usage.Speed } // Carry the sandbox container on the message_delta event so the reverse // converter can re-emit it (Anthropic delivers it here, not earlier). @@ -7816,4 +7817,4 @@ func generateSyntheticInputJSONDeltas(argumentsJSON string, contentIndex *int) [ } return events -} \ No newline at end of file +} diff --git a/core/providers/anthropic/utils_test.go b/core/providers/anthropic/utils_test.go index 9f0f3e4d65c..d3307a0c158 100644 --- a/core/providers/anthropic/utils_test.go +++ b/core/providers/anthropic/utils_test.go @@ -3475,3 +3475,46 @@ func TestStripEmptyThinkingBlocks(t *testing.T) { }) } } + +// TestFastMode_StreamingForwardsSpeed verifies the streaming Responses converter +// forwards the served speed onto the response, so streamed fast-mode requests +// bill at fast rates (parity with the non-streaming ToBifrostResponsesResponse +// path). Without this, tier.isFast is false for streams and cache/input/output +// all bill at standard rates. +func TestFastMode_StreamingForwardsSpeed(t *testing.T) { + ctx := schemas.NewBifrostContext(nil, time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyIntegrationType, "anthropic") + state := AcquireAnthropicResponsesStreamState() + defer ReleaseAnthropicResponsesStreamState(state) + + // Final usage arrives on message_delta: speed:"fast" + 5m cache-creation tokens. + raw := `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":2,"output_tokens":135,"cache_creation_input_tokens":44667,"cache_creation":{"ephemeral_5m_input_tokens":44667,"ephemeral_1h_input_tokens":0},"speed":"fast"}}` + var chunk AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(raw), &chunk); err != nil { + t.Fatalf("unmarshal event: %v", err) + } + + responses, bErr, _ := chunk.ToBifrostResponsesStream(ctx, 0, state) + if bErr != nil { + t.Fatalf("ToBifrostResponsesStream error: %v", bErr) + } + + var sawUsage bool + for _, r := range responses { + if r.Response == nil || r.Response.Usage == nil { + continue + } + sawUsage = true + if r.Response.Speed == nil || *r.Response.Speed != "fast" { + t.Fatalf("streamed message_delta did not forward speed=fast; got %v", r.Response.Speed) + } + // Cache-creation tokens must survive so the fast cache rate applies. + if r.Response.Usage.InputTokensDetails == nil || + r.Response.Usage.InputTokensDetails.CachedWriteTokens != 44667 { + t.Fatalf("cache-creation tokens not carried onto streamed usage") + } + } + if !sawUsage { + t.Fatalf("no usage-bearing response emitted from message_delta") + } +} diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 561b6bdbf33..84a0fcb36fa 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -436,6 +436,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_model_pricing_is_deprecated_column"}, run: migrationAddModelPricingIsDeprecatedColumn}, {IDs: []string{"add_mcp_client_tool_execution_timeout_column"}, run: migrationAddMCPClientToolExecutionTimeoutColumn}, {IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn}, + {IDs: []string{"add_fast_mode_cache_pricing_columns"}, run: migrationAddFastModeCachePricingColumns}, } // quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes. @@ -7797,6 +7798,45 @@ func migrationAddFastModePricingColumns(ctx context.Context, db *gorm.DB, logger return nil } +// migrationAddFastModeCachePricingColumns adds fast-mode cache pricing columns +// for Anthropic (speed:"fast"). Caching multipliers stack on the fast base input +// rate, so cache tokens need dedicated fast rates instead of the standard ones. +func migrationAddFastModeCachePricingColumns(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_fast_mode_cache_pricing_columns" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + columns := []string{ + "cache_creation_input_token_cost_fast", + "cache_creation_input_token_cost_above_1hr_fast", + "cache_read_input_token_cost_fast", + } + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := addColumnIfNotExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to add column %s: %w", field, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := dropColumnIfExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to drop column %s: %w", field, err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while running fast mode cache pricing columns migration: %s", err.Error()) + } + return nil +} + // migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { migrationName := "add_whitelisted_routes_json_column" diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index 524cf1b2cb5..e4aeedc961c 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -2491,6 +2491,9 @@ var pricingSyncUpdateColumns = []string{ "cache_read_input_image_token_cost", "cache_read_input_token_cost_above_272k_tokens", "cache_read_input_token_cost_above_272k_tokens_priority", + "cache_creation_input_token_cost_fast", + "cache_creation_input_token_cost_above_1hr_fast", + "cache_read_input_token_cost_fast", // Costs - Image "input_cost_per_image", "input_cost_per_pixel", diff --git a/framework/configstore/tables/modelpricing.go b/framework/configstore/tables/modelpricing.go index d1c7385dd7b..34d157e80b8 100644 --- a/framework/configstore/tables/modelpricing.go +++ b/framework/configstore/tables/modelpricing.go @@ -30,7 +30,7 @@ type TableModelPricing struct { InputCostPerTokenFlex *float64 `gorm:"default:null;column:input_cost_per_token_flex" json:"input_cost_per_token_flex,omitempty"` OutputCostPerTokenFlex *float64 `gorm:"default:null;column:output_cost_per_token_flex" json:"output_cost_per_token_flex,omitempty"` // Fast mode (Anthropic research preview, speed:"fast" on Opus 4.6/4.7/4.8). - // Flat rate across the full context window; cache tokens bill at standard cache rates. + // Flat rate across the full context window; cache tokens use the _fast cache columns below. InputCostPerTokenFast *float64 `gorm:"default:null;column:input_cost_per_token_fast" json:"input_cost_per_token_fast,omitempty"` OutputCostPerTokenFast *float64 `gorm:"default:null;column:output_cost_per_token_fast" json:"output_cost_per_token_fast,omitempty"` InputCostPerCharacter *float64 `gorm:"default:null;column:input_cost_per_character" json:"input_cost_per_character,omitempty"` @@ -65,6 +65,10 @@ type TableModelPricing struct { CacheReadInputImageTokenCost *float64 `gorm:"default:null;column:cache_read_input_image_token_cost" json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens" json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens_priority" json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. + CacheCreationInputTokenCostFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_fast" json:"cache_creation_input_token_cost_fast,omitempty"` + CacheCreationInputTokenCostAbove1hrFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_above_1hr_fast" json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` + CacheReadInputTokenCostFast *float64 `gorm:"default:null;column:cache_read_input_token_cost_fast" json:"cache_read_input_token_cost_fast,omitempty"` // Costs - Image InputCostPerImage *float64 `gorm:"default:null;column:input_cost_per_image" json:"input_cost_per_image,omitempty"` diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index c7a89b2388d..8e36f108d6c 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -937,6 +937,10 @@ func tieredAudioTokenOutputRate(pricing *configstoreTables.TableModelPricing, to } func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheReadInputTokenCostFast != nil { + return *pricing.CacheReadInputTokenCostFast + } if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil { return *pricing.CacheReadInputTokenCostFlex } @@ -969,6 +973,10 @@ func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, // OpenAI's pricing model (the only provider that uses flex tier). Only cache read // has a flex-specific rate. func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheCreationInputTokenCostFast != nil { + return *pricing.CacheCreationInputTokenCostFast + } if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil { return *pricing.CacheCreationInputTokenCostAbove200kTokens } @@ -979,6 +987,10 @@ func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPric } func tieredCacheCreationInputAbove1hrTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { + // Fast mode (Anthropic) is a flat rate across the full context window. + if tier.isFast && pricing.CacheCreationInputTokenCostAbove1hrFast != nil { + return *pricing.CacheCreationInputTokenCostAbove1hrFast + } if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens != nil { return *pricing.CacheCreationInputTokenCostAbove1hrAbove200kTokens } diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 7c40b590d06..3ddf9befe83 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -205,14 +205,16 @@ func TestComputeTextCost_FastMode_FallsBackWhenUnconfigured(t *testing.T) { assert.InDelta(t, 1000*0.000005+500*0.000025, fast, 1e-12) } -func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) { - // Per design: cache tokens on a fast request bill at standard cache rates; - // only the non-cached input and the output use the fast rate. +func TestComputeTextCost_FastMode_UsesFastCacheRates(t *testing.T) { + // Fast mode has dedicated cache columns; when set, cache tokens bill at the + // _fast rate, not the standard rate. p := chatPricing(0.000005, 0.000025) p.InputCostPerTokenFast = bifrost.Ptr(0.00001) p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) - p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read - p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) // standard read (ignored in fast) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast) + p.CacheReadInputTokenCostFast = bifrost.Ptr(0.000001) // fast read + p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write usage := &schemas.BifrostLLMUsage{ PromptTokens: 2000, @@ -225,12 +227,66 @@ func TestComputeTextCost_FastMode_CacheBillsAtStandardRates(t *testing.T) { } fast := computeTextCost(&p, usage, serviceTier{isFast: true}) - // Input: non-cached (2000-1500-200)*fast + read 1500*stdRead + write 200*stdWrite - // = 300*0.00001 + 1500*0.0000005 + 200*0.00000625 = 0.003 + 0.00075 + 0.00125 = 0.0019(? recompute) + // non-cached 300*fast + read 1500*fastRead + write 200*fastWrite + output 500*fast + expected := 300*0.00001 + 1500*0.000001 + 200*0.0000125 + 500*0.00005 + assert.InDelta(t, expected, fast, 1e-12) +} + +func TestComputeTextCost_FastMode_CacheFallsBackToStandardWhenFastUnset(t *testing.T) { + // When the _fast cache columns are absent, cache tokens fall back to standard + // cache rates (mirrors the input/output fast fallback) — the flag is a no-op. + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 2000, + CompletionTokens: 500, + TotalTokens: 2500, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 1500, + CachedWriteTokens: 200, + }, + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // input/output use fast; cache uses standard. expected := 300*0.00001 + 1500*0.0000005 + 200*0.00000625 + 500*0.00005 assert.InDelta(t, expected, fast, 1e-12) } +// TestComputeTextCost_FastMode_Opus48CacheRegression pins the reported real-world +// miscalculation: fast mode + cache_control billed cache creation at the standard +// rate. Opus 4.8: fast $10/$50, fast 5m cache write $12.50 per MTok. +func TestComputeTextCost_FastMode_Opus48CacheRegression(t *testing.T) { + p := chatPricing(0.000005, 0.000025) + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00005) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) // standard 5m write (ignored in fast) + p.CacheCreationInputTokenCostFast = bifrost.Ptr(0.0000125) // fast 5m write + + // input_tokens=2, cache_creation=44667 (all 5m), output=135. PromptTokens + // carries the cache-creation tokens (Anthropic responses usage mapping). + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 44669, + CompletionTokens: 135, + TotalTokens: 44804, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedWriteTokens: 44667, + CachedWriteTokenDetails: &schemas.ChatCachedWriteTokenDetails{ + CachedWriteTokens5m: 44667, + }, + }, + } + + fast := computeTextCost(&p, usage, serviceTier{isFast: true}) + // 2*$10/M + 44667*$12.50/M (fast 5m cache) + 135*$50/M = $0.565108 + expected := 2*0.00001 + 44667*0.0000125 + 135*0.00005 + assert.InDelta(t, expected, fast, 1e-9) +} + func TestTierFromResponse_Speed(t *testing.T) { assert.False(t, tierFromResponse(nil, nil).isFast) assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard")).isFast) diff --git a/framework/modelcatalog/datasheet/overrides.go b/framework/modelcatalog/datasheet/overrides.go index 67c335e4b7e..84d19c38ea7 100644 --- a/framework/modelcatalog/datasheet/overrides.go +++ b/framework/modelcatalog/datasheet/overrides.go @@ -289,6 +289,9 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options) {dst: &patched.CacheReadInputTokenCostAbove200kTokensPriority, src: override.CacheReadInputTokenCostAbove200kTokensPriority}, {dst: &patched.CacheReadInputTokenCostAbove272kTokens, src: override.CacheReadInputTokenCostAbove272kTokens}, {dst: &patched.CacheReadInputTokenCostAbove272kTokensPriority, src: override.CacheReadInputTokenCostAbove272kTokensPriority}, + {dst: &patched.CacheCreationInputTokenCostFast, src: override.CacheCreationInputTokenCostFast}, + {dst: &patched.CacheCreationInputTokenCostAbove1hrFast, src: override.CacheCreationInputTokenCostAbove1hrFast}, + {dst: &patched.CacheReadInputTokenCostFast, src: override.CacheReadInputTokenCostFast}, {dst: &patched.InputCostPerTokenBatches, src: override.InputCostPerTokenBatches}, {dst: &patched.OutputCostPerTokenBatches, src: override.OutputCostPerTokenBatches}, {dst: &patched.InputCostPerImageToken, src: override.InputCostPerImageToken}, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 2616bf8694f..28182156971 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -139,6 +139,10 @@ type Options struct { CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. + CacheCreationInputTokenCostFast *float64 `json:"cache_creation_input_token_cost_fast,omitempty"` + CacheCreationInputTokenCostAbove1hrFast *float64 `json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` + CacheReadInputTokenCostFast *float64 `json:"cache_read_input_token_cost_fast,omitempty"` // Costs - Image InputCostPerImage *float64 `json:"input_cost_per_image,omitempty"` @@ -593,6 +597,9 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables. CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority, + CacheCreationInputTokenCostFast: entry.CacheCreationInputTokenCostFast, + CacheCreationInputTokenCostAbove1hrFast: entry.CacheCreationInputTokenCostAbove1hrFast, + CacheReadInputTokenCostFast: entry.CacheReadInputTokenCostFast, InputCostPerImage: entry.InputCostPerImage, InputCostPerPixel: entry.InputCostPerPixel, @@ -670,6 +677,9 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority, + CacheCreationInputTokenCostFast: pricing.CacheCreationInputTokenCostFast, + CacheCreationInputTokenCostAbove1hrFast: pricing.CacheCreationInputTokenCostAbove1hrFast, + CacheReadInputTokenCostFast: pricing.CacheReadInputTokenCostFast, InputCostPerImage: pricing.InputCostPerImage, InputCostPerPixel: pricing.InputCostPerPixel, diff --git a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx index 4e4fb5ee703..7c3363d97af 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx +++ b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx @@ -170,6 +170,9 @@ export const PRICING_FIELDS = [ }, { key: "cache_read_input_token_cost_priority", label: "Cache read / token (priority)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_flex", label: "Cache read / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_creation_input_token_cost_fast", label: "Cache creation / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_creation_input_token_cost_above_1hr_fast", label: "Cache creation / token (>1hr, fast)", group: "chat", requestTypeGroups: ["chat"] }, + { key: "cache_read_input_token_cost_fast", label: "Cache read / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_above_200k_tokens_priority", label: "Cache read / token (>200k, priority)", diff --git a/ui/lib/types/governance.ts b/ui/lib/types/governance.ts index 58cdeb83c59..3921cbb4a0e 100644 --- a/ui/lib/types/governance.ts +++ b/ui/lib/types/governance.ts @@ -455,6 +455,9 @@ export interface PricingOverridePatch { cache_read_input_image_token_cost?: number; cache_read_input_token_cost_above_272k_tokens?: number; cache_read_input_token_cost_above_272k_tokens_priority?: number; + cache_creation_input_token_cost_fast?: number; + cache_creation_input_token_cost_above_1hr_fast?: number; + cache_read_input_token_cost_fast?: number; // Image input_cost_per_image_token?: number; output_cost_per_image_token?: number; From 19fb62749b82a86fed9c0b81e573e99833a6f52e Mon Sep 17 00:00:00 2001 From: Tejas Ghatte <64637256+TejasGhatte@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:57:51 +0530 Subject: [PATCH 2/5] fix: inference geo cost on anthropic (#5072) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds support for Anthropic's data-residency billing feature (`inference_geo`). When Anthropic serves inference in the US (`inference_geo: "us"`), a 1.1x multiplier is applied to all token and cache costs. This PR propagates the `inference_geo` field through the full response pipeline (chat, responses, streaming, passthrough) and wires it into the cost computation layer, including a new database column and UI override field. A secondary fix ensures Anthropic server-tool web search request counts (`ServerToolUse.WebSearchRequests`) are correctly forwarded and billed in both streaming and non-streaming paths, where previously the count could be lost when the terminal chunk overwrote the accumulated usage. ## Changes - **`inference_geo` propagation**: `InferenceGeo *string` added to `BifrostChatResponse`, `BifrostResponsesResponse`, and `BifrostPassthroughUsage`. The field is forwarded from `AnthropicUsage` in all conversion paths: non-streaming chat, non-streaming responses, streaming chat (captured across events and set on the final chunk), streaming responses (`message_delta`), and passthrough. - **Data-residency multiplier in cost computation**: `serviceTier` gains an `inferenceGeoUS bool` flag. `tierFromResponse` now accepts and evaluates `inferenceGeo`. `computeTextCost` applies `InferenceGeoUSMultiplier` to token/cache costs only — the flat per-search fee is intentionally excluded. - **Database migration**: `migrationAddInferenceGeoMultiplierColumn` adds `inference_geo_us_multiplier` to `TableModelPricing`. The column is included in the pricing sync update list and mapped through `convertEntryToTablePricing` / `convertTablePricingToEntry`. - **Web search billing fix**: `accumulateAnthropicResponsesUsage` now carries `ServerToolUse.WebSearchRequests` into the accumulator so the count survives the terminal-chunk overwrite on streamed Responses requests. The non-streaming chat converter also forwards the count into `CompletionTokensDetails.NumSearchQueries`. - **UI**: `inference_geo_us_multiplier` added to the custom pricing override sheet and `PricingOverridePatch` TypeScript type. - **Tests**: New unit tests cover the accumulator web-search fix, chat/responses converter forwarding of `InferenceGeo` and web search counts, multiplier application (including the no-multiplier-column safe no-op), and `tierFromResponse` geo detection (case-insensitive). ## Type of change - [ ] Bug fix - [x] Feature - [ ] Refactor - [ ] Documentation - [ ] Chore/CI ## Affected areas - [x] Core (Go) - [ ] Transports (HTTP) - [x] Providers/Integrations - [ ] Plugins - [x] UI (React) - [ ] Docs ## How to test ```sh # Core/Transports go test ./core/providers/anthropic/... ./framework/modelcatalog/datasheet/... # UI cd ui pnpm i || npm i pnpm build || npm run build ``` To validate end-to-end: send a request to an Anthropic model with data residency enabled. The response should include `inference_geo: "us"` and the computed cost should reflect the 1.1x multiplier on token/cache costs, with the per-search fee unchanged. ## Breaking changes - [ ] Yes - [x] No ## Security considerations No auth, secrets, or PII implications. The `inference_geo` value is sourced from Anthropic's API response and used only for billing computation. ## Checklist - [ ] I read `docs/contributing/README.md` and followed the guidelines - [x] I added/updated tests where appropriate - [ ] I updated documentation where needed - [x] I verified builds succeed (Go and UI) - [ ] I verified the CI pipeline passes locally if applicable --- core/providers/anthropic/anthropic.go | 73 +++++++ core/providers/anthropic/chat.go | 11 + core/providers/anthropic/passthrough_usage.go | 6 + core/providers/anthropic/responses.go | 6 + core/providers/anthropic/utils_test.go | 206 +++++++++++++++++- core/schemas/chatcompletions.go | 10 +- core/schemas/passthrough.go | 7 +- core/schemas/responses.go | 9 +- framework/configstore/migrations.go | 38 ++++ framework/configstore/rdb.go | 1 + framework/configstore/tables/modelpricing.go | 2 + framework/modelcatalog/datasheet/cost.go | 28 ++- framework/modelcatalog/datasheet/cost_test.go | 92 +++++++- framework/modelcatalog/datasheet/overrides.go | 1 + framework/modelcatalog/datasheet/types.go | 5 + .../overrides/pricingOverrideSheet.tsx | 1 + ui/lib/types/governance.ts | 1 + 17 files changed, 469 insertions(+), 28 deletions(-) diff --git a/core/providers/anthropic/anthropic.go b/core/providers/anthropic/anthropic.go index a5e44a272fc..e5a069de4ff 100644 --- a/core/providers/anthropic/anthropic.go +++ b/core/providers/anthropic/anthropic.go @@ -621,6 +621,26 @@ func accumulateAnthropicResponsesUsage(usage *schemas.ResponsesResponseUsage, bi if usage == nil || usageToProcess == nil { return } + // Web search request count → billed as search queries (server tool use). The + // terminal chunk overwrites Response.Usage with this accumulator, so the count + // must live here (not only on the per-event message_delta usage). + if usageToProcess.ServerToolUse != nil && usageToProcess.ServerToolUse.WebSearchRequests > 0 { + n := usageToProcess.ServerToolUse.WebSearchRequests + if usage.OutputTokensDetails == nil { + usage.OutputTokensDetails = &schemas.ResponsesResponseOutputTokens{} + } + if usage.OutputTokensDetails.NumSearchQueries == nil || n > *usage.OutputTokensDetails.NumSearchQueries { + usage.OutputTokensDetails.NumSearchQueries = schemas.Ptr(n) + } + if billedUsage != nil { + if billedUsage.CompletionTokensDetails == nil { + billedUsage.CompletionTokensDetails = &schemas.ChatCompletionTokensDetails{} + } + if billedUsage.CompletionTokensDetails.NumSearchQueries == nil || n > *billedUsage.CompletionTokensDetails.NumSearchQueries { + billedUsage.CompletionTokensDetails.NumSearchQueries = schemas.Ptr(n) + } + } + } if usageToProcess.InputTokens > usage.InputTokens { usage.InputTokens = usageToProcess.InputTokens if billedUsage != nil { @@ -863,6 +883,10 @@ func HandleAnthropicChatCompletionStreaming( var finishReason *string usage := &schemas.BifrostLLMUsage{} + // Served billing modifiers (top-level response fields, not usage) captured + // across events and set on the final chunk: fast mode and data residency. + var servedSpeed *string + var servedInferenceGeo *string // Register the accumulating usage handle so a mid-stream cancel/timeout // can bill for tokens already processed Mutated in place below; // the deferred HandleStreamCancellation/Timeout reads it from context. @@ -938,6 +962,26 @@ func HandleAnthropicChatCompletionStreaming( usageToProcess = event.Message.Usage } if usageToProcess != nil { + // Web search request count → billed as search queries (server tool use). + if usageToProcess.ServerToolUse != nil && usageToProcess.ServerToolUse.WebSearchRequests > 0 { + if usage.CompletionTokensDetails == nil { + usage.CompletionTokensDetails = &schemas.ChatCompletionTokensDetails{} + } + if n := usageToProcess.ServerToolUse.WebSearchRequests; usage.CompletionTokensDetails.NumSearchQueries == nil || n > *usage.CompletionTokensDetails.NumSearchQueries { + usage.CompletionTokensDetails.NumSearchQueries = &n + } + } + // Capture served fast mode + inference geography (top-level response fields). + // Mirror onto the billing usage handle so a mid-stream cancel/timeout can + // still apply the served-tier multiplier (billed usage is otherwise bare). + if usageToProcess.Speed != nil { + servedSpeed = usageToProcess.Speed + usage.Speed = usageToProcess.Speed + } + if usageToProcess.InferenceGeo != nil { + servedInferenceGeo = usageToProcess.InferenceGeo + usage.InferenceGeo = usageToProcess.InferenceGeo + } // Collect usage information and send at the end of the stream // Here in some cases usage comes before final message // So we need to check if the response.Usage is nil and then if usage != nil @@ -1099,6 +1143,13 @@ func HandleAnthropicChatCompletionStreaming( return } } + // Forward served fast mode + data residency so the final chunk bills correctly. + if servedSpeed != nil { + response.Speed = servedSpeed + } + if servedInferenceGeo != nil { + response.InferenceGeo = servedInferenceGeo + } // Set raw request if enabled if sendBackRawRequest { providerUtils.ParseAndSetRawRequest(&response.ExtraFields, jsonBody) @@ -1455,6 +1506,8 @@ func HandleAnthropicResponsesStream( } var modelName string + var servedSpeed *string + var servedInferenceGeo *string for { // If context was cancelled/timed out, let defer handle it @@ -1503,6 +1556,20 @@ func HandleAnthropicResponsesStream( // Also mirror it into billedUsage so cancellation/timeout paths can // charge for provider-reported usage before the final chunk arrives. accumulateAnthropicResponsesUsage(usage, billedUsage, usageToProcess) + // Mirror served tier onto billedUsage so a mid-stream cancel/timeout can + // still apply the served-tier multiplier (billed usage is otherwise bare). + if usageToProcess.Speed != nil { + servedSpeed = usageToProcess.Speed + if billedUsage != nil { + billedUsage.Speed = usageToProcess.Speed + } + } + if usageToProcess.InferenceGeo != nil { + servedInferenceGeo = usageToProcess.InferenceGeo + if billedUsage != nil { + billedUsage.InferenceGeo = usageToProcess.InferenceGeo + } + } } responses, bifrostErr, isLastChunk := event.ToBifrostResponsesStream(ctx, chunkIndex, streamState) @@ -1561,6 +1628,12 @@ func HandleAnthropicResponsesStream( usage.TotalTokens = usage.TotalTokens + usage.InputTokensDetails.CachedReadTokens + usage.InputTokensDetails.CachedWriteTokens } response.Response.Usage = usage + if servedSpeed != nil { + response.Response.Speed = servedSpeed + } + if servedInferenceGeo != nil { + response.Response.InferenceGeo = servedInferenceGeo + } // Set raw request if enabled if sendBackRawRequest { providerUtils.ParseAndSetRawRequest(&response.ExtraFields, jsonBody) diff --git a/core/providers/anthropic/chat.go b/core/providers/anthropic/chat.go index 600f5fc8547..4e0d141fcb2 100644 --- a/core/providers/anthropic/chat.go +++ b/core/providers/anthropic/chat.go @@ -1033,6 +1033,13 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif PromptTokensDetails: promptTokensDetails, CompletionTokens: response.Usage.OutputTokens, } + // Forward web search request count so server-tool use is billed. + if response.Usage.ServerToolUse != nil && response.Usage.ServerToolUse.WebSearchRequests > 0 { + n := response.Usage.ServerToolUse.WebSearchRequests + bifrostResponse.Usage.CompletionTokensDetails = &schemas.ChatCompletionTokensDetails{ + NumSearchQueries: &n, + } + } bifrostResponse.Usage.TotalTokens = bifrostResponse.Usage.PromptTokens + bifrostResponse.Usage.CompletionTokens // Forward service tier from usage to response if response.Usage.ServiceTier != nil { @@ -1043,6 +1050,10 @@ func (response *AnthropicMessageResponse) ToBifrostChatResponse(ctx *schemas.Bif if response.Usage.Speed != nil { bifrostResponse.Speed = response.Usage.Speed } + // Forward the inference geography served — drives the data-residency multiplier. + if response.Usage.InferenceGeo != nil { + bifrostResponse.InferenceGeo = response.Usage.InferenceGeo + } } // Forward cache diagnostics (cache-diagnosis-2026-04-07) — top-level on the diff --git a/core/providers/anthropic/passthrough_usage.go b/core/providers/anthropic/passthrough_usage.go index bddf96dd42d..aef6fc3fed9 100644 --- a/core/providers/anthropic/passthrough_usage.go +++ b/core/providers/anthropic/passthrough_usage.go @@ -77,6 +77,9 @@ func buildAnthropicPassthroughUsage(au *AnthropicUsage) *schemas.BifrostPassthro if au.Speed != nil { u.Speed = au.Speed } + if au.InferenceGeo != nil { + u.InferenceGeo = au.InferenceGeo + } return u } @@ -142,6 +145,9 @@ func (a *AnthropicPassthroughStreamUsage) ObserveEvent(event []byte) *schemas.Bi if u.Speed != nil { c.Speed = u.Speed } + if u.InferenceGeo != nil { + c.InferenceGeo = u.InferenceGeo + } return a.usage() } diff --git a/core/providers/anthropic/responses.go b/core/providers/anthropic/responses.go index c1a9034e8a1..c0310e8ee6b 100644 --- a/core/providers/anthropic/responses.go +++ b/core/providers/anthropic/responses.go @@ -1989,6 +1989,7 @@ func (chunk *AnthropicStreamEvent) ToBifrostResponsesStream(ctx context.Context, if bifrostUsage != nil { response.Usage = bifrostUsage response.Speed = chunk.Usage.Speed + response.InferenceGeo = chunk.Usage.InferenceGeo } // Carry the sandbox container on the message_delta event so the reverse // converter can re-emit it (Anthropic delivers it here, not earlier). @@ -3703,6 +3704,11 @@ func (response *AnthropicMessageResponse) ToBifrostResponsesResponse(ctx *schema bifrostResp.Speed = response.Usage.Speed } + // Forward the inference geography served — drives the data-residency multiplier. + if response.Usage != nil && response.Usage.InferenceGeo != nil { + bifrostResp.InferenceGeo = response.Usage.InferenceGeo + } + // Forward cache diagnostics (cache-diagnosis-2026-04-07) to the client. if response.Diagnostics != nil { bifrostResp.Diagnostics = response.Diagnostics diff --git a/core/providers/anthropic/utils_test.go b/core/providers/anthropic/utils_test.go index d3307a0c158..7ea4682d554 100644 --- a/core/providers/anthropic/utils_test.go +++ b/core/providers/anthropic/utils_test.go @@ -3476,11 +3476,11 @@ func TestStripEmptyThinkingBlocks(t *testing.T) { } } -// TestFastMode_StreamingForwardsSpeed verifies the streaming Responses converter -// forwards the served speed onto the response, so streamed fast-mode requests -// bill at fast rates (parity with the non-streaming ToBifrostResponsesResponse -// path). Without this, tier.isFast is false for streams and cache/input/output -// all bill at standard rates. +// TestFastMode_StreamingForwardsSpeed verifies the per-event message_delta +// converter surfaces the served speed on the emitted chunk (client-facing usage +// visibility). NOTE: billing reads the terminal response.completed chunk, not +// message_delta — that end-to-end billing contract is covered by +// TestResponsesStream_TerminalChunkCarriesServedModifiers. func TestFastMode_StreamingForwardsSpeed(t *testing.T) { ctx := schemas.NewBifrostContext(nil, time.Time{}) ctx.SetValue(schemas.BifrostContextKeyIntegrationType, "anthropic") @@ -3518,3 +3518,199 @@ func TestFastMode_StreamingForwardsSpeed(t *testing.T) { t.Fatalf("no usage-bearing response emitted from message_delta") } } + +// TestAccumulateResponsesUsage_BillsWebSearch verifies the streaming Responses +// usage accumulator carries server-tool web search counts onto both the response +// usage and the mirrored billed usage. The terminal chunk overwrites +// Response.Usage with this accumulator, so without this the per-event search count +// is lost and web search goes unbilled on streamed Responses requests. +func TestAccumulateResponsesUsage_BillsWebSearch(t *testing.T) { + usage := &schemas.ResponsesResponseUsage{} + billed := &schemas.BifrostLLMUsage{} + accumulateAnthropicResponsesUsage(usage, billed, &AnthropicUsage{ + InputTokens: 105, + OutputTokens: 6039, + ServerToolUse: &AnthropicServerToolUseUsage{WebSearchRequests: 2}, + }) + + if usage.OutputTokensDetails == nil || usage.OutputTokensDetails.NumSearchQueries == nil { + t.Fatal("response usage NumSearchQueries not set") + } + if got := *usage.OutputTokensDetails.NumSearchQueries; got != 2 { + t.Fatalf("response usage NumSearchQueries = %d, want 2", got) + } + if billed.CompletionTokensDetails == nil || billed.CompletionTokensDetails.NumSearchQueries == nil { + t.Fatal("billed usage NumSearchQueries not set") + } + if got := *billed.CompletionTokensDetails.NumSearchQueries; got != 2 { + t.Fatalf("billed usage NumSearchQueries = %d, want 2", got) + } +} + +// TestToBifrostChatResponse_ForwardsWebSearchAndInferenceGeo verifies the chat +// converter surfaces server-tool web search counts (so they bill at +// search_context_cost_per_query) and forwards the served inference geography (so +// the data-residency multiplier applies) alongside fast-mode speed. +func TestToBifrostChatResponse_ForwardsWebSearchAndInferenceGeo(t *testing.T) { + response := &AnthropicMessageResponse{ + ID: "msg_ws", + Type: "message", + Role: "assistant", + Model: "claude-opus-4-8", + Content: []AnthropicContentBlock{ + {Type: AnthropicContentBlockTypeText, Text: schemas.Ptr("hi")}, + }, + StopReason: AnthropicStopReasonEndTurn, + Usage: &AnthropicUsage{ + InputTokens: 105, + OutputTokens: 6039, + ServerToolUse: &AnthropicServerToolUseUsage{WebSearchRequests: 3}, + InferenceGeo: schemas.Ptr("us"), + Speed: schemas.Ptr("fast"), + }, + } + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + + result := response.ToBifrostChatResponse(ctx) + if result == nil || result.Usage == nil { + t.Fatal("expected non-nil result with usage") + } + if result.Usage.CompletionTokensDetails == nil || result.Usage.CompletionTokensDetails.NumSearchQueries == nil { + t.Fatal("web search request count not forwarded to chat usage") + } + if got := *result.Usage.CompletionTokensDetails.NumSearchQueries; got != 3 { + t.Fatalf("chat usage NumSearchQueries = %d, want 3", got) + } + if result.InferenceGeo == nil || *result.InferenceGeo != "us" { + t.Fatalf("inference_geo not forwarded; got %v", result.InferenceGeo) + } + if result.Speed == nil || *result.Speed != "fast" { + t.Fatalf("speed not forwarded; got %v", result.Speed) + } +} + +// TestToBifrostResponsesResponse_ForwardsInferenceGeo verifies the non-streaming +// Responses converter forwards the served inference geography for data-residency +// billing (parity with the streaming message_delta path). +func TestToBifrostResponsesResponse_ForwardsInferenceGeo(t *testing.T) { + response := &AnthropicMessageResponse{ + ID: "msg_geo", + Type: "message", + Role: "assistant", + Model: "claude-opus-4-8", + Content: []AnthropicContentBlock{ + {Type: AnthropicContentBlockTypeText, Text: schemas.Ptr("hi")}, + }, + StopReason: AnthropicStopReasonEndTurn, + Usage: &AnthropicUsage{ + InputTokens: 10, + OutputTokens: 5, + InferenceGeo: schemas.Ptr("us"), + }, + } + ctx, cancel := schemas.NewBifrostContextWithCancel(context.Background()) + defer cancel() + + result := response.ToBifrostResponsesResponse(ctx) + if result == nil { + t.Fatal("expected non-nil result") + } + if result.InferenceGeo == nil || *result.InferenceGeo != "us" { + t.Fatalf("inference_geo not forwarded; got %v", result.InferenceGeo) + } +} + +// TestResponsesStream_TerminalChunkCarriesServedModifiers pins the streaming +// Responses BILLING contract. Billing (framework/streaming/responses.go) prices +// the terminal response.completed chunk — whose builder starts fresh with no +// Speed/InferenceGeo/Usage. So the handler must (a) accumulate usage across events +// and (b) re-apply the served fast mode + data residency captured from earlier +// events onto that terminal chunk. This replays message_start → message_delta → +// message_stop through the real converters + accumulator and reproduces the +// handler's capture/apply, asserting the billed chunk carries speed=fast, +// inference_geo=us, the web-search count, and the cache-creation tokens. Without +// the re-apply, speed/geo silently fall back to standard/non-US rates. +func TestResponsesStream_TerminalChunkCarriesServedModifiers(t *testing.T) { + ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) + ctx.SetValue(schemas.BifrostContextKeyIntegrationType, "anthropic") + state := AcquireAnthropicResponsesStreamState() + defer ReleaseAnthropicResponsesStreamState(state) + + usage := &schemas.ResponsesResponseUsage{} + billed := &schemas.BifrostLLMUsage{} + var servedSpeed, servedInferenceGeo *string + + events := []string{ + `{"type":"message_start","message":{"id":"msg_1","model":"claude-opus-4-8","usage":{"input_tokens":2,"cache_creation_input_tokens":44667,"cache_creation":{"ephemeral_5m_input_tokens":44667}}}}`, + `{"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"input_tokens":2,"output_tokens":135,"cache_creation_input_tokens":44667,"cache_creation":{"ephemeral_5m_input_tokens":44667},"server_tool_use":{"web_search_requests":4},"speed":"fast","inference_geo":"us"}}`, + `{"type":"message_stop"}`, + } + + var finalResp *schemas.BifrostResponsesResponse + for _, raw := range events { + var event AnthropicStreamEvent + if err := sonic.Unmarshal([]byte(raw), &event); err != nil { + t.Fatalf("unmarshal: %v", err) + } + // Handler step 1: extract usage (top-level or nested), accumulate, capture + // served modifiers — unconditionally, mirroring HandleAnthropicResponsesStream. + var usageToProcess *AnthropicUsage + if event.Usage != nil { + usageToProcess = event.Usage + } else if event.Message != nil && event.Message.Usage != nil { + usageToProcess = event.Message.Usage + } + if usageToProcess != nil { + accumulateAnthropicResponsesUsage(usage, billed, usageToProcess) + if usageToProcess.Speed != nil { + servedSpeed = usageToProcess.Speed + } + if usageToProcess.InferenceGeo != nil { + servedInferenceGeo = usageToProcess.InferenceGeo + } + } + // Handler step 2: convert + on the terminal chunk, attach usage and re-apply + // the captured served modifiers. + responses, bErr, isLastChunk := event.ToBifrostResponsesStream(ctx, 0, state) + if bErr != nil { + t.Fatalf("ToBifrostResponsesStream: %v", bErr) + } + if isLastChunk && len(responses) > 0 { + r := responses[len(responses)-1] + if r.Response == nil { + r.Response = &schemas.BifrostResponsesResponse{} + } + // Contract precondition: response.completed starts fresh (no served fields). + if r.Response.Speed != nil || r.Response.InferenceGeo != nil { + t.Fatal("expected fresh response.completed with no served modifiers") + } + r.Response.Usage = usage + if servedSpeed != nil { + r.Response.Speed = servedSpeed + } + if servedInferenceGeo != nil { + r.Response.InferenceGeo = servedInferenceGeo + } + finalResp = r.Response + } + } + + if finalResp == nil { + t.Fatal("no terminal (isLastChunk) response produced") + } + if finalResp.Speed == nil || *finalResp.Speed != "fast" { + t.Fatalf("terminal billed chunk missing speed=fast; got %v", finalResp.Speed) + } + if finalResp.InferenceGeo == nil || *finalResp.InferenceGeo != "us" { + t.Fatalf("terminal billed chunk missing inference_geo=us; got %v", finalResp.InferenceGeo) + } + if finalResp.Usage == nil || finalResp.Usage.OutputTokensDetails == nil || + finalResp.Usage.OutputTokensDetails.NumSearchQueries == nil || + *finalResp.Usage.OutputTokensDetails.NumSearchQueries != 4 { + t.Fatal("terminal billed chunk missing web search count") + } + if finalResp.Usage.InputTokensDetails == nil || finalResp.Usage.InputTokensDetails.CachedWriteTokens != 44667 { + t.Fatal("terminal billed chunk missing cache-creation tokens") + } +} diff --git a/core/schemas/chatcompletions.go b/core/schemas/chatcompletions.go index 96f3fd1028c..ca750eafce4 100644 --- a/core/schemas/chatcompletions.go +++ b/core/schemas/chatcompletions.go @@ -40,8 +40,9 @@ type BifrostChatResponse struct { Model string `json:"model"` Object string `json:"object"` // "chat.completion" or "chat.completion.chunk" ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` - Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing - Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point + Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing + InferenceGeo *string `json:"inference_geo,omitempty"` // "us" | "global" — inference geography served (Anthropic data residency); drives the 1.1x US multiplier + Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point SystemFingerprint string `json:"system_fingerprint"` Usage *BifrostLLMUsage `json:"usage"` ExtraFields BifrostResponseExtraFields `json:"extra_fields"` @@ -1621,6 +1622,11 @@ type BifrostLLMUsage struct { CompletionTokensDetails *ChatCompletionTokensDetails `json:"completion_tokens_details,omitempty"` TotalTokens int `json:"total_tokens"` Cost *BifrostCost `json:"cost,omitempty"` // Only for the providers which support cost calculation + // Served Anthropic tier (fast mode / data residency), carried internally so + // cancel/timeout billing (which reads a bare usage via BilledUsage) can apply + // the tier multiplier. json:"-" keeps them out of every serialized usage payload. + Speed *string `json:"-"` + InferenceGeo *string `json:"-"` } type ChatPromptTokensDetails struct { diff --git a/core/schemas/passthrough.go b/core/schemas/passthrough.go index 12ef96508bb..2f48289014f 100644 --- a/core/schemas/passthrough.go +++ b/core/schemas/passthrough.go @@ -15,9 +15,10 @@ type BifrostPassthroughRequest struct { // functions — no new pricing logic is required. type BifrostPassthroughUsage struct { // Text / chat / responses / embeddings - LLMUsage *BifrostLLMUsage - ServiceTier *BifrostServiceTier // "priority" | "flex" | nil (default) - Speed *string // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing + LLMUsage *BifrostLLMUsage + ServiceTier *BifrostServiceTier // "priority" | "flex" | nil (default) + Speed *string // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing + InferenceGeo *string // "us" | "global" — inference geography served (Anthropic data residency); drives the 1.1x US multiplier // Image generation / edit / variation ImageUsage *ImageUsage diff --git a/core/schemas/responses.go b/core/schemas/responses.go index 7119f7b748a..d7bef7308b0 100644 --- a/core/schemas/responses.go +++ b/core/schemas/responses.go @@ -230,10 +230,11 @@ type BifrostResponsesResponse struct { Reasoning *ResponsesParametersReasoning `json:"reasoning"` // Configuration options for reasoning models SafetyIdentifier *string `json:"safety_identifier"` // Safety identifier ServiceTier *BifrostServiceTier `json:"service_tier"` - Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing - Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point - Container *ResponsesResponseContainer `json:"container,omitempty"` // Code-execution sandbox container (Anthropic surfaces it on the response / final streaming message_delta). The neutral per-call id also lives on ResponsesCodeInterpreterToolCall.ContainerID. - Status *string `json:"status,omitempty"` // completed, failed, in_progress, cancelled, queued, or incomplete + Speed *string `json:"speed,omitempty"` // "fast" | "standard" — speed actually served (Anthropic fast mode); drives fast-mode billing + InferenceGeo *string `json:"inference_geo,omitempty"` // "us" | "global" — inference geography served (Anthropic data residency); drives the 1.1x US multiplier + Diagnostics *CacheDiagnostics `json:"diagnostics,omitempty"` // Anthropic cache diagnostics (cache-diagnosis-2026-04-07); first prompt-cache prefix divergence point + Container *ResponsesResponseContainer `json:"container,omitempty"` // Code-execution sandbox container (Anthropic surfaces it on the response / final streaming message_delta). The neutral per-call id also lives on ResponsesCodeInterpreterToolCall.ContainerID. + Status *string `json:"status,omitempty"` // completed, failed, in_progress, cancelled, queued, or incomplete StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` StopReason *string `json:"stop_reason,omitempty"` // Not in OpenAI's spec, but sent by other providers Store *bool `json:"store,omitempty"` diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index 84a0fcb36fa..e57d28028c5 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -437,6 +437,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_mcp_client_tool_execution_timeout_column"}, run: migrationAddMCPClientToolExecutionTimeoutColumn}, {IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn}, {IDs: []string{"add_fast_mode_cache_pricing_columns"}, run: migrationAddFastModeCachePricingColumns}, + {IDs: []string{"add_inference_geo_multiplier_column"}, run: migrationAddInferenceGeoMultiplierColumn}, } // quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes. @@ -7837,6 +7838,43 @@ func migrationAddFastModeCachePricingColumns(ctx context.Context, db *gorm.DB, l return nil } +// migrationAddInferenceGeoMultiplierColumn adds the inference_geo_us_multiplier +// column for Anthropic data residency (inference_geo:"us" applies a 1.1x +// multiplier stacking on top of all token/cache costs). +func migrationAddInferenceGeoMultiplierColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_inference_geo_multiplier_column" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + columns := []string{ + "inference_geo_us_multiplier", + } + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := addColumnIfNotExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to add column %s: %w", field, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := dropColumnIfExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to drop column %s: %w", field, err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while running inference geo multiplier column migration: %s", err.Error()) + } + return nil +} + // migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { migrationName := "add_whitelisted_routes_json_column" diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index e4aeedc961c..60e36069516 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -2494,6 +2494,7 @@ var pricingSyncUpdateColumns = []string{ "cache_creation_input_token_cost_fast", "cache_creation_input_token_cost_above_1hr_fast", "cache_read_input_token_cost_fast", + "inference_geo_us_multiplier", // Costs - Image "input_cost_per_image", "input_cost_per_pixel", diff --git a/framework/configstore/tables/modelpricing.go b/framework/configstore/tables/modelpricing.go index 34d157e80b8..251bbf5014e 100644 --- a/framework/configstore/tables/modelpricing.go +++ b/framework/configstore/tables/modelpricing.go @@ -101,6 +101,8 @@ type TableModelPricing struct { // Costs - Other SearchContextCostPerQuery *float64 `gorm:"default:null;column:search_context_cost_per_query" json:"search_context_cost_per_query,omitempty"` CodeInterpreterCostPerSession *float64 `gorm:"default:null;column:code_interpreter_cost_per_session" json:"code_interpreter_cost_per_session,omitempty"` + // Data-residency multiplier scaling all token/cache costs when Anthropic serves inference_geo:"us" (1.1x); nil = no multiplier. + InferenceGeoUSMultiplier *float64 `gorm:"default:null;column:inference_geo_us_multiplier" json:"inference_geo_us_multiplier,omitempty"` // Costs - OCR OCRCostPerPage *float64 `gorm:"default:null;column:ocr_cost_per_page" json:"ocr_cost_per_page,omitempty"` diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index 8e36f108d6c..2394c87e44e 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -55,8 +55,13 @@ func (s *Store) CalculateCostForUsage(usage *schemas.BifrostLLMUsage, provider s return usage.Cost.TotalCost } + // Apply the served tier (fast mode / data residency) carried on the usage so + // cancelled/failed fast or US-residency streams keep their multiplier. + input := costInput{usage: usage} + input.tier = tierFromResponse(nil, usage.Speed, usage.InferenceGeo) + return s.computeCostFromInput( - costInput{usage: usage}, + input, schemas.RoutingInfo{Provider: provider, Model: model}, normalizeStreamRequestType(requestType), lookupScopes, @@ -225,18 +230,18 @@ func extractCostInput(result *schemas.BifrostResponse) costInput { case result.ChatResponse != nil && result.ChatResponse.Usage != nil: input.usage = result.ChatResponse.Usage - input.tier = tierFromResponse(result.ChatResponse.ServiceTier, result.ChatResponse.Speed) + input.tier = tierFromResponse(result.ChatResponse.ServiceTier, result.ChatResponse.Speed, result.ChatResponse.InferenceGeo) case result.ResponsesResponse != nil && result.ResponsesResponse.Usage != nil: input.usage = responsesUsageToBifrostUsage(result.ResponsesResponse.Usage) - input.tier = tierFromResponse(result.ResponsesResponse.ServiceTier, result.ResponsesResponse.Speed) + input.tier = tierFromResponse(result.ResponsesResponse.ServiceTier, result.ResponsesResponse.Speed, result.ResponsesResponse.InferenceGeo) case result.CompactionResponse != nil && result.CompactionResponse.Usage != nil: input.usage = responsesUsageToBifrostUsage(result.CompactionResponse.Usage) case result.ResponsesStreamResponse != nil && result.ResponsesStreamResponse.Response != nil && result.ResponsesStreamResponse.Response.Usage != nil: input.usage = responsesUsageToBifrostUsage(result.ResponsesStreamResponse.Response.Usage) - input.tier = tierFromResponse(result.ResponsesStreamResponse.Response.ServiceTier, result.ResponsesStreamResponse.Response.Speed) + input.tier = tierFromResponse(result.ResponsesStreamResponse.Response.ServiceTier, result.ResponsesStreamResponse.Response.Speed, result.ResponsesStreamResponse.Response.InferenceGeo) case result.EmbeddingResponse != nil && result.EmbeddingResponse.Usage != nil: input.usage = result.EmbeddingResponse.Usage @@ -478,7 +483,15 @@ func computeTextCost(pricing *configstoreTables.TableModelPricing, usage *schema searchCost = float64(*usage.CompletionTokensDetails.NumSearchQueries) * *pricing.SearchContextCostPerQuery } - return inputCost + outputCost + audioCost + searchCost + // Data residency (Anthropic inference_geo:"us") scales all token/cache costs + // by a flat multiplier; the per-search fee is not a token category, so it is + // excluded. + tokenCost := inputCost + outputCost + audioCost + if tier.inferenceGeoUS && pricing.InferenceGeoUSMultiplier != nil { + tokenCost *= *pricing.InferenceGeoUSMultiplier + } + + return tokenCost + searchCost } // computeEmbeddingCost handles embedding requests (input-only). @@ -777,7 +790,7 @@ func computeOCRCost(pricing *configstoreTables.TableModelPricing, ocrProcessedPa // (fast mode). speed == "fast" means fast mode was actually served — the // provider echoes the served speed, so stripped/fell-back requests report // "standard" and bill at standard rates. -func tierFromResponse(s *schemas.BifrostServiceTier, speed *string) serviceTier { +func tierFromResponse(s *schemas.BifrostServiceTier, speed *string, inferenceGeo *string) serviceTier { var tier serviceTier if s != nil { switch *s { @@ -788,6 +801,7 @@ func tierFromResponse(s *schemas.BifrostServiceTier, speed *string) serviceTier } } tier.isFast = speed != nil && *speed == "fast" + tier.inferenceGeoUS = inferenceGeo != nil && strings.EqualFold(*inferenceGeo, "us") return tier } @@ -1419,7 +1433,7 @@ func passthroughUsageToCostInput(su *schemas.BifrostPassthroughUsage) costInput if su.LLMUsage != nil { input.usage = su.LLMUsage } - input.tier = tierFromResponse(su.ServiceTier, su.Speed) + input.tier = tierFromResponse(su.ServiceTier, su.Speed, su.InferenceGeo) if su.ImageUsage != nil { input.imageUsage = su.ImageUsage input.imageSize = su.ImageSize diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 3ddf9befe83..1892946874b 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -287,10 +287,60 @@ func TestComputeTextCost_FastMode_Opus48CacheRegression(t *testing.T) { assert.InDelta(t, expected, fast, 1e-9) } +// TestComputeTextCost_InferenceGeoUS_AppliesMultiplier verifies the Anthropic +// data-residency multiplier (inference_geo:"us") scales every token/cache cost by +// 1.1x while leaving the flat per-search fee untouched. +func TestComputeTextCost_InferenceGeoUS_AppliesMultiplier(t *testing.T) { + p := chatPricing(0.00001, 0.00005) + p.CacheReadInputTokenCost = bifrost.Ptr(0.000001) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.0000125) + p.SearchContextCostPerQuery = bifrost.Ptr(0.01) + p.InferenceGeoUSMultiplier = bifrost.Ptr(1.1) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 1000, // 500 non-cached + 200 read + 300 write + CompletionTokens: 100, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 200, + CachedWriteTokens: 300, + }, + CompletionTokensDetails: &schemas.ChatCompletionTokensDetails{ + NumSearchQueries: bifrost.Ptr(2), + }, + } + + tokenCost := 500*0.00001 + 200*0.000001 + 300*0.0000125 + 100*0.00005 + searchCost := 2 * 0.01 + + got := computeTextCost(&p, usage, serviceTier{inferenceGeoUS: true}) + assert.InDelta(t, tokenCost*1.1+searchCost, got, 1e-9) + + // Without US residency the multiplier is a no-op; the search fee is identical. + base := computeTextCost(&p, usage, serviceTier{}) + assert.InDelta(t, tokenCost+searchCost, base, 1e-9) +} + +// TestComputeTextCost_InferenceGeoUS_NoMultiplierColumn verifies US residency is a +// safe no-op until the datasheet populates the multiplier column upstream. +func TestComputeTextCost_InferenceGeoUS_NoMultiplierColumn(t *testing.T) { + p := chatPricing(0.00001, 0.00005) + usage := &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 100} + withUS := computeTextCost(&p, usage, serviceTier{inferenceGeoUS: true}) + without := computeTextCost(&p, usage, serviceTier{}) + assert.InDelta(t, without, withUS, 1e-9) +} + func TestTierFromResponse_Speed(t *testing.T) { - assert.False(t, tierFromResponse(nil, nil).isFast) - assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard")).isFast) - assert.True(t, tierFromResponse(nil, bifrost.Ptr("fast")).isFast) + assert.False(t, tierFromResponse(nil, nil, nil).isFast) + assert.False(t, tierFromResponse(nil, bifrost.Ptr("standard"), nil).isFast) + assert.True(t, tierFromResponse(nil, bifrost.Ptr("fast"), nil).isFast) +} + +func TestTierFromResponse_InferenceGeo(t *testing.T) { + assert.False(t, tierFromResponse(nil, nil, nil).inferenceGeoUS) + assert.False(t, tierFromResponse(nil, nil, bifrost.Ptr("global")).inferenceGeoUS) + assert.True(t, tierFromResponse(nil, nil, bifrost.Ptr("us")).inferenceGeoUS) + assert.True(t, tierFromResponse(nil, nil, bifrost.Ptr("US")).inferenceGeoUS) } func TestComputeTextCost_With1hrCacheCreationTokens(t *testing.T) { @@ -2351,28 +2401,28 @@ func TestTieredCacheReadRate_FallbackOrder(t *testing.T) { func TestTierFromResponse_Priority(t *testing.T) { s := schemas.BifrostServiceTierPriority - tier := tierFromResponse(&s, nil) + tier := tierFromResponse(&s, nil, nil) assert.True(t, tier.isPriority) assert.False(t, tier.isFlex) } func TestTierFromResponse_Flex(t *testing.T) { s := schemas.BifrostServiceTierFlex - tier := tierFromResponse(&s, nil) + tier := tierFromResponse(&s, nil, nil) assert.False(t, tier.isPriority) assert.True(t, tier.isFlex) } func TestTierFromResponse_Default(t *testing.T) { for _, s := range []schemas.BifrostServiceTier{schemas.BifrostServiceTierAuto, schemas.BifrostServiceTierDefault, ""} { - tier := tierFromResponse(&s, nil) + tier := tierFromResponse(&s, nil, nil) assert.False(t, tier.isPriority, "expected no priority for %q", s) assert.False(t, tier.isFlex, "expected no flex for %q", s) } } func TestTierFromResponse_Nil(t *testing.T) { - tier := tierFromResponse(nil, nil) + tier := tierFromResponse(nil, nil, nil) assert.False(t, tier.isPriority) assert.False(t, tier.isFlex) } @@ -3100,3 +3150,31 @@ func TestCalculateCostForUsage_NilUsageIsZero(t *testing.T) { }) assert.Equal(t, 0.0, s.CalculateCostForUsage(nil, schemas.OpenAI, "gpt-4o", schemas.ChatCompletionRequest, nil)) } + +// TestCalculateCostForUsage_AppliesServedTier verifies the cancel/failure billing +// path honors the served Anthropic tier carried internally on the usage (fast mode +// + data residency), so interrupted fast or US-residency streams keep their +// multiplier instead of being billed as standard/global. +func TestCalculateCostForUsage_AppliesServedTier(t *testing.T) { + p := chatPricing(0.000005, 0.000015) // base $5/$15 per MTok + p.InputCostPerTokenFast = bifrost.Ptr(0.00001) + p.OutputCostPerTokenFast = bifrost.Ptr(0.00003) + p.InferenceGeoUSMultiplier = bifrost.Ptr(1.1) + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("claude-x", "anthropic", "chat"): p, + }) + mk := func(speed, geo *string) *schemas.BifrostLLMUsage { + return &schemas.BifrostLLMUsage{PromptTokens: 1000, CompletionTokens: 500, TotalTokens: 1500, Speed: speed, InferenceGeo: geo} + } + cost := func(u *schemas.BifrostLLMUsage) float64 { + return s.CalculateCostForUsage(u, schemas.Anthropic, "claude-x", schemas.ChatCompletionRequest, nil) + } + fast, us := "fast", "us" + + // No served tier → base rate. + assert.InDelta(t, 1000*0.000005+500*0.000015, cost(mk(nil, nil)), 1e-12) + // speed:"fast" → flat fast rate. + assert.InDelta(t, 1000*0.00001+500*0.00003, cost(mk(&fast, nil)), 1e-12) + // inference_geo:"us" → 1.1x on base. + assert.InDelta(t, (1000*0.000005+500*0.000015)*1.1, cost(mk(nil, &us)), 1e-12) +} diff --git a/framework/modelcatalog/datasheet/overrides.go b/framework/modelcatalog/datasheet/overrides.go index 84d19c38ea7..72151224d6f 100644 --- a/framework/modelcatalog/datasheet/overrides.go +++ b/framework/modelcatalog/datasheet/overrides.go @@ -292,6 +292,7 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options) {dst: &patched.CacheCreationInputTokenCostFast, src: override.CacheCreationInputTokenCostFast}, {dst: &patched.CacheCreationInputTokenCostAbove1hrFast, src: override.CacheCreationInputTokenCostAbove1hrFast}, {dst: &patched.CacheReadInputTokenCostFast, src: override.CacheReadInputTokenCostFast}, + {dst: &patched.InferenceGeoUSMultiplier, src: override.InferenceGeoUSMultiplier}, {dst: &patched.InputCostPerTokenBatches, src: override.InputCostPerTokenBatches}, {dst: &patched.OutputCostPerTokenBatches, src: override.OutputCostPerTokenBatches}, {dst: &patched.InputCostPerImageToken, src: override.InputCostPerImageToken}, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 28182156971..9078d64a776 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -178,6 +178,7 @@ type Options struct { // represents it as a tiered object. See Entry.UnmarshalJSON. SearchContextCostPerQuery *float64 `json:"search_context_cost_per_query,omitempty"` CodeInterpreterCostPerSession *float64 `json:"code_interpreter_cost_per_session,omitempty"` + InferenceGeoUSMultiplier *float64 `json:"inference_geo_us_multiplier,omitempty"` // Costs - OCR OCRCostPerPage *float64 `json:"ocr_cost_per_page,omitempty"` @@ -256,6 +257,8 @@ type serviceTier struct { isPriority bool // true when service_tier == "priority" isFlex bool // true when service_tier == "flex" isFast bool // true when usage.speed == "fast" (Anthropic fast mode) + // true when usage.inference_geo == "us" (Anthropic data residency 1.1x multiplier) + inferenceGeoUS bool } // costInput holds the extracted usage data from a BifrostResponse, @@ -629,6 +632,7 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables. SearchContextCostPerQuery: entry.SearchContextCostPerQuery, CodeInterpreterCostPerSession: entry.CodeInterpreterCostPerSession, + InferenceGeoUSMultiplier: entry.InferenceGeoUSMultiplier, OCRCostPerPage: entry.OCRCostPerPage, AnnotationCostPerPage: entry.AnnotationCostPerPage, @@ -709,6 +713,7 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E SearchContextCostPerQuery: pricing.SearchContextCostPerQuery, CodeInterpreterCostPerSession: pricing.CodeInterpreterCostPerSession, + InferenceGeoUSMultiplier: pricing.InferenceGeoUSMultiplier, OCRCostPerPage: pricing.OCRCostPerPage, AnnotationCostPerPage: pricing.AnnotationCostPerPage, diff --git a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx index 7c3363d97af..1cb147af6b6 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx +++ b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx @@ -188,6 +188,7 @@ export const PRICING_FIELDS = [ }, { key: "search_context_cost_per_query", label: "Search context / query", group: "chat", requestTypeGroups: ["chat", "rerank"] }, { key: "code_interpreter_cost_per_session", label: "Code interpreter / session", group: "chat", requestTypeGroups: ["chat"] }, + { key: "inference_geo_us_multiplier", label: "Inference geo US multiplier", group: "chat", requestTypeGroups: ["chat"] }, // Audio fields { key: "input_cost_per_character", label: "Input / character", group: "audio", requestTypeGroups: ["audio"] }, { key: "input_cost_per_audio_token", label: "Input / audio token", group: "audio", requestTypeGroups: ["audio"] }, diff --git a/ui/lib/types/governance.ts b/ui/lib/types/governance.ts index 3921cbb4a0e..971fde9c24e 100644 --- a/ui/lib/types/governance.ts +++ b/ui/lib/types/governance.ts @@ -485,6 +485,7 @@ export interface PricingOverridePatch { // Other search_context_cost_per_query?: number; code_interpreter_cost_per_session?: number; + inference_geo_us_multiplier?: number; // OCR ocr_cost_per_page?: number; annotation_cost_per_page?: number; From c898ed975e023f0f2662564c734215b5301389c3 Mon Sep 17 00:00:00 2001 From: Tejas Ghatte <64637256+TejasGhatte@users.noreply.github.com> Date: Thu, 9 Jul 2026 21:58:51 +0530 Subject: [PATCH 3/5] fix: service tier in openai chat completion (#5073) ## Summary During OpenAI chat completion streaming, the `service_tier` field returned in stream chunks was being discarded, causing the final assembled response to lose priority/flex billing tier information. This PR captures `service_tier` as it appears in stream chunks and propagates it to the final response. ## Changes - Introduced a `serviceTier` variable to track the `service_tier` value echoed across streaming chunks. - After the stream completes and the final response is assembled, the captured `service_tier` is applied to the response so priority/flex billing metadata is preserved. ## Type of change - [x] Bug fix - [ ] Feature - [ ] Refactor - [ ] Documentation - [ ] Chore/CI ## Affected areas - [ ] Core (Go) - [ ] Transports (HTTP) - [x] Providers/Integrations - [ ] Plugins - [ ] UI (React) - [ ] Docs ## How to test Send a streaming chat completion request to an OpenAI endpoint using a `service_tier` (e.g., `"flex"` or `"priority"`). Verify that the final streamed response includes the correct `service_tier` value. ```sh go test ./... ``` ## Breaking changes - [ ] Yes - [x] No ## Related issues ## Security considerations No security implications. This change only propagates a billing tier metadata field from stream chunks to the final response object. ## Checklist - [ ] I read `docs/contributing/README.md` and followed the guidelines - [ ] I added/updated tests where appropriate - [ ] I updated documentation where needed - [ ] I verified builds succeed (Go and UI) - [ ] I verified the CI pipeline passes locally if applicable --- core/providers/openai/openai.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/providers/openai/openai.go b/core/providers/openai/openai.go index 4230bf7c6a1..b5b40158167 100644 --- a/core/providers/openai/openai.go +++ b/core/providers/openai/openai.go @@ -1174,6 +1174,8 @@ func HandleOpenAIChatCompletionStreaming( var messageID string var modelName string var created int + // service_tier is echoed on chunks; propagate to the final chunk for priority/flex billing + var serviceTier *schemas.BifrostServiceTier forwardedTerminalFinishReason := false // Defer final completed/incomplete event until usage chunk arrives (fallback path only). var pendingFinalEvent *schemas.BifrostResponsesStreamResponse @@ -1316,6 +1318,10 @@ func HandleOpenAIChatCompletionStreaming( } } + if response.ServiceTier != nil { + serviceTier = response.ServiceTier + } + // Handle usage-only chunks (when stream_options include_usage is true) if response.Usage != nil { // Collect usage information and send at the end of the stream @@ -1422,6 +1428,10 @@ func HandleOpenAIChatCompletionStreaming( if postResponseConverter != nil { response = postResponseConverter(response) } + // Preserve captured tier so priority/flex billing applies to the streamed response + if serviceTier != nil { + response.ServiceTier = serviceTier + } // Set raw request if enabled if sendBackRawRequest { providerUtils.ParseAndSetRawRequest(&response.ExtraFields, jsonBody) From b61075562a84065e4d6c17c8e8ee84de96c58c56 Mon Sep 17 00:00:00 2001 From: tejas ghatte Date: Fri, 10 Jul 2026 11:26:32 +0530 Subject: [PATCH 4/5] fix: openai cache write pricings --- core/mcp/agentadaptors.go | 1 + core/providers/openai/chat.go | 3 + core/providers/openai/chat_test.go | 37 ++ core/providers/openai/responses.go | 3 + core/schemas/chatcompletions.go | 13 + core/schemas/mux.go | 2 + core/schemas/responses.go | 35 +- core/schemas/serialization_test.go | 120 +++++ framework/configstore/migrations.go | 44 ++ framework/configstore/rdb.go | 7 + framework/configstore/tables/modelpricing.go | 8 + framework/modelcatalog/datasheet/cost.go | 51 ++- framework/modelcatalog/datasheet/cost_test.go | 425 ++++++++++++++++++ framework/modelcatalog/datasheet/overrides.go | 7 + framework/modelcatalog/datasheet/types.go | 22 + .../overrides/pricingOverrideSheet.tsx | 44 +- ui/lib/types/governance.ts | 7 + 17 files changed, 817 insertions(+), 12 deletions(-) diff --git a/core/mcp/agentadaptors.go b/core/mcp/agentadaptors.go index 389c845d672..7d641bdb1d9 100644 --- a/core/mcp/agentadaptors.go +++ b/core/mcp/agentadaptors.go @@ -449,6 +449,7 @@ func createResponsesResponseWithExecutedToolsAndNonAutoExecutableCalls( Prompt: originalResponse.Prompt, PromptCacheKey: originalResponse.PromptCacheKey, PromptCacheRetention: originalResponse.PromptCacheRetention, + PromptCacheOptions: originalResponse.PromptCacheOptions, Reasoning: originalResponse.Reasoning, SafetyIdentifier: originalResponse.SafetyIdentifier, ServiceTier: originalResponse.ServiceTier, diff --git a/core/providers/openai/chat.go b/core/providers/openai/chat.go index 147fb2dbc27..8b62c4e4e8c 100644 --- a/core/providers/openai/chat.go +++ b/core/providers/openai/chat.go @@ -136,6 +136,9 @@ func (req *OpenAIChatRequest) filterOpenAISpecificParameters(capModel string) { if req.ChatParameters.PromptCacheRetention != nil { req.ChatParameters.PromptCacheRetention = nil } + if req.ChatParameters.PromptCacheOptions != nil { + req.ChatParameters.PromptCacheOptions = nil + } if req.ChatParameters.Verbosity != nil { req.ChatParameters.Verbosity = nil } diff --git a/core/providers/openai/chat_test.go b/core/providers/openai/chat_test.go index 9fafa75b510..3debb7af8a3 100644 --- a/core/providers/openai/chat_test.go +++ b/core/providers/openai/chat_test.go @@ -588,6 +588,43 @@ func TestToOpenAIChatRequest_CachingDeterminism(t *testing.T) { } } +func TestToOpenAIChatRequest_PromptCacheOptions(t *testing.T) { + ctx, cancel := schemas.NewBifrostContextWithCancel(nil) + defer cancel() + + mode := "explicit" + ttl := "30m" + userContent := "hello" + mkReq := func(provider schemas.ModelProvider, model string) *schemas.BifrostChatRequest { + return &schemas.BifrostChatRequest{ + Provider: provider, + Model: model, + Input: []schemas.ChatMessage{{ + Role: schemas.ChatMessageRoleUser, + Content: &schemas.ChatMessageContent{ContentStr: &userContent}, + }}, + Params: &schemas.ChatParameters{ + PromptCacheOptions: &schemas.PromptCacheOptions{Mode: &mode, TTL: &ttl}, + }, + } + } + + // OpenAI keeps the OpenAI-native field. + openai := ToOpenAIChatRequest(ctx, mkReq(schemas.OpenAI, "gpt-5.6")) + if openai == nil || openai.ChatParameters.PromptCacheOptions == nil { + t.Fatal("expected prompt_cache_options preserved for OpenAI") + } + if *openai.ChatParameters.PromptCacheOptions.Mode != mode || *openai.ChatParameters.PromptCacheOptions.TTL != ttl { + t.Fatalf("unexpected options: %#v", openai.ChatParameters.PromptCacheOptions) + } + + // A non-OpenAI OpenAI-compatible provider strips it. + fw := ToOpenAIChatRequest(ctx, mkReq(schemas.Fireworks, "accounts/fireworks/models/deepseek-v3p2")) + if fw == nil || fw.ChatParameters.PromptCacheOptions != nil { + t.Fatalf("expected prompt_cache_options stripped for Fireworks, got %#v", fw.ChatParameters.PromptCacheOptions) + } +} + func TestToOpenAIChatRequest_FireworksPreservesReasoningAndCacheIsolation(t *testing.T) { ctx, cancel := schemas.NewBifrostContextWithCancel(nil) defer cancel() diff --git a/core/providers/openai/responses.go b/core/providers/openai/responses.go index 152fcc5a3c4..450348969e8 100644 --- a/core/providers/openai/responses.go +++ b/core/providers/openai/responses.go @@ -437,6 +437,7 @@ type OpenAICompactionRequest struct { PreviousResponseID *string `json:"previous_response_id,omitempty"` PromptCacheKey *string `json:"prompt_cache_key,omitempty"` PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` + PromptCacheOptions *schemas.PromptCacheOptions `json:"prompt_cache_options,omitempty"` ServiceTier *schemas.BifrostServiceTier `json:"service_tier,omitempty"` ExtraParams map[string]interface{} `json:"-"` } @@ -455,6 +456,7 @@ func ToOpenAICompactionRequest(ctx *schemas.BifrostContext, req *schemas.Bifrost PreviousResponseID: req.PreviousResponseID, PromptCacheKey: req.PromptCacheKey, PromptCacheRetention: req.PromptCacheRetention, + PromptCacheOptions: req.PromptCacheOptions, ServiceTier: req.ServiceTier, ExtraParams: req.ExtraParams, } @@ -497,6 +499,7 @@ func (r *OpenAICompactionRequest) ToBifrostCompactionRequest(ctx *schemas.Bifros PreviousResponseID: r.PreviousResponseID, PromptCacheKey: r.PromptCacheKey, PromptCacheRetention: r.PromptCacheRetention, + PromptCacheOptions: r.PromptCacheOptions, ServiceTier: r.ServiceTier, ExtraParams: r.ExtraParams, } diff --git a/core/schemas/chatcompletions.go b/core/schemas/chatcompletions.go index ca750eafce4..0aac2e434cb 100644 --- a/core/schemas/chatcompletions.go +++ b/core/schemas/chatcompletions.go @@ -204,6 +204,7 @@ type ChatParameters struct { PresencePenalty *float64 `json:"presence_penalty,omitempty"` // Penalizes repeated tokens PromptCacheKey *string `json:"prompt_cache_key,omitempty"` // Prompt cache key PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` // Prompt cache retention ("in_memory" or "24h") + PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` // Request-wide prompt cache options (OpenAI gpt-5.6+) Reasoning *ChatReasoning `json:"reasoning,omitempty"` // Reasoning parameters ResponseFormat *interface{} `json:"response_format,omitempty"` // Format for the response SafetyIdentifier *string `json:"safety_identifier,omitempty"` // Safety identifier @@ -1124,6 +1125,9 @@ type ChatContentBlock struct { CacheControl *CacheControl `json:"cache_control,omitempty"` Citations *Citations `json:"citations,omitempty"` + // PromptCacheBreakpoint marks an explicit prompt-cache breakpoint on this block (OpenAI gpt-5.6+). + PromptCacheBreakpoint *PromptCacheBreakpoint `json:"prompt_cache_breakpoint,omitempty"` + // CachePoint is a Bedrock-specific field for standalone cache point blocks // When present without other content, this indicates a cache point marker CachePoint *CachePoint `json:"cachePoint,omitempty"` @@ -1655,6 +1659,7 @@ func (d *ChatPromptTokensDetails) UnmarshalJSON(data []byte) error { CachedWriteTokens int `json:"cached_write_tokens"` CachedWriteTokenDetails *ChatCachedWriteTokenDetails `json:"cached_write_token_details"` CachedTokens *int `json:"cached_tokens"` + CacheWriteTokens *int `json:"cache_write_tokens"` } if err := Unmarshal(data, &raw); err != nil { return err @@ -1669,6 +1674,10 @@ func (d *ChatPromptTokensDetails) UnmarshalJSON(data []byte) error { if raw.CachedTokens != nil && raw.CachedReadTokens == 0 && raw.CachedWriteTokens == 0 { d.CachedReadTokens = *raw.CachedTokens } + // OpenAI's Responses API reports cache writes under cache_write_tokens (distinct from Bifrost's cached_write_tokens). + if raw.CacheWriteTokens != nil && d.CachedWriteTokens == 0 { + d.CachedWriteTokens = *raw.CacheWriteTokens + } return nil } @@ -1684,6 +1693,9 @@ func (d ChatPromptTokensDetails) MarshalJSON() ([]byte, error) { CachedWriteTokens int `json:"cached_write_tokens,omitempty"` CachedWriteTokenDetails *ChatCachedWriteTokenDetails `json:"cached_write_token_details,omitempty"` CachedTokens int `json:"cached_tokens"` + // OpenAI's field name for cache writes (mirrors cached_tokens for reads) so the + // OpenAI SDK — which reads cache_write_tokens, not cached_write_tokens — finds it. + CacheWriteTokens int `json:"cache_write_tokens"` } return MarshalSorted(raw{ TextTokens: d.TextTokens, @@ -1693,6 +1705,7 @@ func (d ChatPromptTokensDetails) MarshalJSON() ([]byte, error) { CachedWriteTokens: d.CachedWriteTokens, CachedWriteTokenDetails: d.CachedWriteTokenDetails, CachedTokens: d.CachedReadTokens, + CacheWriteTokens: d.CachedWriteTokens, }) } diff --git a/core/schemas/mux.go b/core/schemas/mux.go index b979cf29353..6678fdecd4b 100644 --- a/core/schemas/mux.go +++ b/core/schemas/mux.go @@ -1078,6 +1078,7 @@ func (cr *BifrostChatRequest) ToResponsesRequest() *BifrostResponsesRequest { ParallelToolCalls: cr.Params.ParallelToolCalls, PromptCacheKey: cr.Params.PromptCacheKey, PromptCacheRetention: cr.Params.PromptCacheRetention, + PromptCacheOptions: cr.Params.PromptCacheOptions, SafetyIdentifier: cr.Params.SafetyIdentifier, ServiceTier: cr.Params.ServiceTier, Store: cr.Params.Store, @@ -1193,6 +1194,7 @@ func (brr *BifrostResponsesRequest) ToChatRequest() *BifrostChatRequest { ParallelToolCalls: brr.Params.ParallelToolCalls, PromptCacheKey: brr.Params.PromptCacheKey, PromptCacheRetention: brr.Params.PromptCacheRetention, + PromptCacheOptions: brr.Params.PromptCacheOptions, SafetyIdentifier: brr.Params.SafetyIdentifier, ServiceTier: brr.Params.ServiceTier, Store: brr.Params.Store, diff --git a/core/schemas/responses.go b/core/schemas/responses.go index d7bef7308b0..dbfc98802b8 100644 --- a/core/schemas/responses.go +++ b/core/schemas/responses.go @@ -150,6 +150,7 @@ type BifrostCompactionRequest struct { PreviousResponseID *string `json:"previous_response_id,omitempty"` PromptCacheKey *string `json:"prompt_cache_key,omitempty"` PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` + PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` Fallbacks []Fallback `json:"fallbacks,omitempty"` ExtraParams map[string]interface{} `json:"-"` @@ -225,6 +226,7 @@ type BifrostResponsesResponse struct { Prompt *ResponsesPrompt `json:"prompt,omitempty"` // Reference to a prompt template and variables PromptCacheKey *string `json:"prompt_cache_key"` // Prompt cache key PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` + PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` // Prompt-caching options applied to the response (OpenAI gpt-5.6+) PresencePenalty *float64 `json:"presence_penalty,omitempty"` FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` Reasoning *ResponsesParametersReasoning `json:"reasoning"` // Configuration options for reasoning models @@ -340,6 +342,7 @@ func (resp *BifrostResponsesResponse) WithDefaults() *BifrostResponsesResponse { result.PreviousResponseID = resp.PreviousResponseID result.PromptCacheKey = resp.PromptCacheKey result.PromptCacheRetention = resp.PromptCacheRetention + result.PromptCacheOptions = resp.PromptCacheOptions result.SafetyIdentifier = resp.SafetyIdentifier result.MaxToolCalls = resp.MaxToolCalls result.Instructions = resp.Instructions @@ -468,6 +471,21 @@ func orDefault[T any](src *T, defaultVal T) *T { return Ptr(defaultVal) } +// PromptCacheOptions is the request-wide prompt-caching configuration OpenAI +// added with the gpt-5.6 family (echoed back on the response). Mode is +// "implicit" or "explicit"; TTL is the minimum breakpoint lifetime (currently +// "30m"). Values are passed through untouched. +type PromptCacheOptions struct { + Mode *string `json:"mode,omitempty"` + TTL *string `json:"ttl,omitempty"` +} + +// PromptCacheBreakpoint marks the end of a cacheable prompt prefix on a content +// block (OpenAI gpt-5.6+). Only "explicit" is valid for Mode. +type PromptCacheBreakpoint struct { + Mode *string `json:"mode,omitempty"` +} + type ResponsesParameters struct { Background *bool `json:"background,omitempty"` Conversation *string `json:"conversation,omitempty"` @@ -480,8 +498,9 @@ type ResponsesParameters struct { PreviousResponseID *string `json:"previous_response_id,omitempty"` PromptCacheKey *string `json:"prompt_cache_key,omitempty"` // Prompt cache key PromptCacheRetention *string `json:"prompt_cache_retention,omitempty"` - Reasoning *ResponsesParametersReasoning `json:"reasoning,omitempty"` // Configuration options for reasoning models - SafetyIdentifier *string `json:"safety_identifier,omitempty"` // Safety identifier + PromptCacheOptions *PromptCacheOptions `json:"prompt_cache_options,omitempty"` // Request-wide prompt cache options (OpenAI gpt-5.6+) + Reasoning *ResponsesParametersReasoning `json:"reasoning,omitempty"` // Configuration options for reasoning models + SafetyIdentifier *string `json:"safety_identifier,omitempty"` // Safety identifier ServiceTier *BifrostServiceTier `json:"service_tier,omitempty"` StreamOptions *ResponsesStreamOptions `json:"stream_options,omitempty"` Store *bool `json:"store,omitempty"` @@ -966,6 +985,7 @@ func (d *ResponsesResponseInputTokens) UnmarshalJSON(data []byte) error { CachedWriteTokens int `json:"cached_write_tokens"` CachedWriteTokenDetails *ChatCachedWriteTokenDetails `json:"cached_write_token_details"` CachedTokens *int `json:"cached_tokens"` + CacheWriteTokens *int `json:"cache_write_tokens"` } if err := Unmarshal(data, &raw); err != nil { return err @@ -980,6 +1000,10 @@ func (d *ResponsesResponseInputTokens) UnmarshalJSON(data []byte) error { if raw.CachedTokens != nil && raw.CachedReadTokens == 0 && raw.CachedWriteTokens == 0 { d.CachedReadTokens = *raw.CachedTokens } + // OpenAI's Responses API reports cache writes under cache_write_tokens (distinct from Bifrost's cached_write_tokens). + if raw.CacheWriteTokens != nil && d.CachedWriteTokens == 0 { + d.CachedWriteTokens = *raw.CacheWriteTokens + } return nil } @@ -995,6 +1019,9 @@ func (d ResponsesResponseInputTokens) MarshalJSON() ([]byte, error) { CachedWriteTokens int `json:"cached_write_tokens"` CachedWriteTokenDetails *ChatCachedWriteTokenDetails `json:"cached_write_token_details,omitempty"` CachedTokens int `json:"cached_tokens"` + // OpenAI's field name for cache writes (mirrors cached_tokens for reads) so the + // OpenAI SDK — which reads cache_write_tokens, not cached_write_tokens — finds it. + CacheWriteTokens int `json:"cache_write_tokens"` } return MarshalSorted(raw{ TextTokens: d.TextTokens, @@ -1004,6 +1031,7 @@ func (d ResponsesResponseInputTokens) MarshalJSON() ([]byte, error) { CachedWriteTokens: d.CachedWriteTokens, CachedWriteTokenDetails: d.CachedWriteTokenDetails, CachedTokens: d.CachedReadTokens, + CacheWriteTokens: d.CachedWriteTokens, }) } @@ -1266,6 +1294,9 @@ type ResponsesMessageContentBlock struct { // Not in OpenAI's schemas, but sent by a few providers (Anthropic, Bedrock are some of them) CacheControl *CacheControl `json:"cache_control,omitempty"` Citations *Citations `json:"citations,omitempty"` + + // PromptCacheBreakpoint marks an explicit prompt-cache breakpoint on this block (OpenAI gpt-5.6+). + PromptCacheBreakpoint *PromptCacheBreakpoint `json:"prompt_cache_breakpoint,omitempty"` } type ResponsesOutputMessageContentCompaction struct { diff --git a/core/schemas/serialization_test.go b/core/schemas/serialization_test.go index e46bc8b7117..72c48a4524b 100644 --- a/core/schemas/serialization_test.go +++ b/core/schemas/serialization_test.go @@ -1558,6 +1558,7 @@ func TestSonic_ChatPromptTokensDetails_CachedTokensExcludesWrites(t *testing.T) require.NoError(t, json.Unmarshal(out, &m)) assert.Equal(t, float64(0), m["cached_tokens"]) assert.Equal(t, float64(9106), m["cached_write_tokens"]) + assert.Equal(t, float64(9106), m["cache_write_tokens"]) // OpenAI SDK reads this name // Cache-hit turn with a concurrent write: cached_tokens must equal reads only. out, err = Marshal(ChatPromptTokensDetails{CachedReadTokens: 500, CachedWriteTokens: 100}) @@ -1567,6 +1568,7 @@ func TestSonic_ChatPromptTokensDetails_CachedTokensExcludesWrites(t *testing.T) assert.Equal(t, float64(500), m["cached_tokens"]) assert.Equal(t, float64(500), m["cached_read_tokens"]) assert.Equal(t, float64(100), m["cached_write_tokens"]) + assert.Equal(t, float64(100), m["cache_write_tokens"]) } func TestSonic_ChatPromptTokensDetails_CachedTokensRoundTrip(t *testing.T) { @@ -1595,6 +1597,7 @@ func TestSonic_ResponsesResponseInputTokens_CachedTokensExcludesWrites(t *testin require.NoError(t, json.Unmarshal(out, &m)) assert.Equal(t, float64(0), m["cached_tokens"]) assert.Equal(t, float64(9106), m["cached_write_tokens"]) + assert.Equal(t, float64(9106), m["cache_write_tokens"]) // OpenAI SDK reads this name // Cache-hit turn with a concurrent write: cached_tokens must equal reads only. out, err = Marshal(ResponsesResponseInputTokens{CachedReadTokens: 500, CachedWriteTokens: 100}) @@ -1604,6 +1607,7 @@ func TestSonic_ResponsesResponseInputTokens_CachedTokensExcludesWrites(t *testin assert.Equal(t, float64(500), m["cached_tokens"]) assert.Equal(t, float64(500), m["cached_read_tokens"]) assert.Equal(t, float64(100), m["cached_write_tokens"]) + assert.Equal(t, float64(100), m["cache_write_tokens"]) } func TestSonic_ResponsesResponseInputTokens_CachedTokensRoundTrip(t *testing.T) { @@ -1620,3 +1624,119 @@ func TestSonic_ResponsesResponseInputTokens_CachedTokensRoundTrip(t *testing.T) assert.Equal(t, 42, d.CachedReadTokens) assert.Equal(t, 0, d.CachedWriteTokens) } + +// OpenAI's Responses API reports cache writes under cache_write_tokens (distinct +// from Bifrost's cached_write_tokens); it must map into CachedWriteTokens. +func TestSonic_ResponsesResponseInputTokens_OpenAICacheWriteTokensAlias(t *testing.T) { + // Fresh-cache Responses turn: OpenAI sends cache_write_tokens with cached_tokens:0. + var d ResponsesResponseInputTokens + require.NoError(t, Unmarshal([]byte(`{"cached_tokens":0,"cache_write_tokens":28003}`), &d)) + assert.Equal(t, 28003, d.CachedWriteTokens) + assert.Equal(t, 0, d.CachedReadTokens) + + // Bifrost's own cached_write_tokens takes precedence when both are present. + var d2 ResponsesResponseInputTokens + require.NoError(t, Unmarshal([]byte(`{"cached_write_tokens":100,"cache_write_tokens":28003}`), &d2)) + assert.Equal(t, 100, d2.CachedWriteTokens) +} + +func TestSonic_ChatPromptTokensDetails_OpenAICacheWriteTokensAlias(t *testing.T) { + var d ChatPromptTokensDetails + require.NoError(t, Unmarshal([]byte(`{"cached_tokens":0,"cache_write_tokens":28003}`), &d)) + assert.Equal(t, 28003, d.CachedWriteTokens) + assert.Equal(t, 0, d.CachedReadTokens) + + var d2 ChatPromptTokensDetails + require.NoError(t, Unmarshal([]byte(`{"cached_write_tokens":100,"cache_write_tokens":28003}`), &d2)) + assert.Equal(t, 100, d2.CachedWriteTokens) +} + +// --- prompt_cache_options / prompt_cache_breakpoint (OpenAI gpt-5.6+) --- + +func TestSonic_PromptCacheOptions_RoundTrip(t *testing.T) { + mode := "explicit" + ttl := "30m" + + // Responses request params serialize the object to the wire. + out, err := Marshal(ResponsesParameters{PromptCacheOptions: &PromptCacheOptions{Mode: &mode, TTL: &ttl}}) + require.NoError(t, err) + var m map[string]any + require.NoError(t, json.Unmarshal(out, &m)) + pco, ok := m["prompt_cache_options"].(map[string]any) + require.True(t, ok, "prompt_cache_options must serialize on ResponsesParameters") + assert.Equal(t, "explicit", pco["mode"]) + assert.Equal(t, "30m", pco["ttl"]) + + // Chat request params round-trip (through ChatParameters' custom unmarshaler). + out, err = Marshal(ChatParameters{PromptCacheOptions: &PromptCacheOptions{Mode: &mode, TTL: &ttl}}) + require.NoError(t, err) + var cp ChatParameters + require.NoError(t, Unmarshal(out, &cp)) + require.NotNil(t, cp.PromptCacheOptions) + assert.Equal(t, "explicit", *cp.PromptCacheOptions.Mode) + assert.Equal(t, "30m", *cp.PromptCacheOptions.TTL) + + // Response echo: OpenAI returns prompt_cache_options on the response object. + var resp BifrostResponsesResponse + require.NoError(t, Unmarshal([]byte(`{"prompt_cache_options":{"mode":"implicit","ttl":"30m"}}`), &resp)) + require.NotNil(t, resp.PromptCacheOptions) + assert.Equal(t, "implicit", *resp.PromptCacheOptions.Mode) + assert.Equal(t, "30m", *resp.PromptCacheOptions.TTL) +} + +func TestSonic_PromptCacheBreakpoint_RoundTrip(t *testing.T) { + mode := "explicit" + + // Responses content block serializes the breakpoint to the wire. + out, err := Marshal(ResponsesMessageContentBlock{ + Type: ResponsesInputMessageContentBlockTypeText, + PromptCacheBreakpoint: &PromptCacheBreakpoint{Mode: &mode}, + }) + require.NoError(t, err) + var m map[string]any + require.NoError(t, json.Unmarshal(out, &m)) + bp, ok := m["prompt_cache_breakpoint"].(map[string]any) + require.True(t, ok, "prompt_cache_breakpoint must serialize on ResponsesMessageContentBlock") + assert.Equal(t, "explicit", bp["mode"]) + + // Chat content block round-trips through its custom unmarshaler. + out, err = Marshal(ChatContentBlock{ + Type: ChatContentBlockTypeText, + PromptCacheBreakpoint: &PromptCacheBreakpoint{Mode: &mode}, + }) + require.NoError(t, err) + var cb ChatContentBlock + require.NoError(t, Unmarshal(out, &cb)) + require.NotNil(t, cb.PromptCacheBreakpoint) + assert.Equal(t, "explicit", *cb.PromptCacheBreakpoint.Mode) +} + +// A native Responses stream's response.completed event carries the full response +// object; prompt_cache_options and cache_write_tokens usage must survive parsing. +func TestSonic_ResponsesStreamCompleted_CapturesPromptCacheAndCacheWrite(t *testing.T) { + event := `{ + "type": "response.completed", + "sequence_number": 42, + "response": { + "id": "resp_1", + "object": "response", + "prompt_cache_options": {"mode": "implicit", "ttl": "30m"}, + "usage": { + "input_tokens": 2006, + "input_tokens_details": {"cached_tokens": 1920, "cache_write_tokens": 80}, + "output_tokens": 300, + "total_tokens": 2306 + } + } + }` + var stream BifrostResponsesStreamResponse + require.NoError(t, Unmarshal([]byte(event), &stream)) + require.NotNil(t, stream.Response) + require.NotNil(t, stream.Response.PromptCacheOptions) + assert.Equal(t, "implicit", *stream.Response.PromptCacheOptions.Mode) + assert.Equal(t, "30m", *stream.Response.PromptCacheOptions.TTL) + require.NotNil(t, stream.Response.Usage) + require.NotNil(t, stream.Response.Usage.InputTokensDetails) + assert.Equal(t, 80, stream.Response.Usage.InputTokensDetails.CachedWriteTokens) + assert.Equal(t, 1920, stream.Response.Usage.InputTokensDetails.CachedReadTokens) +} diff --git a/framework/configstore/migrations.go b/framework/configstore/migrations.go index e57d28028c5..93b7bf2caf8 100644 --- a/framework/configstore/migrations.go +++ b/framework/configstore/migrations.go @@ -438,6 +438,7 @@ var configstoreMigrationSteps = []migrationStep{ {IDs: []string{"add_virtual_key_expires_at_column"}, run: migrationAddVirtualKeyExpiresAtColumn}, {IDs: []string{"add_fast_mode_cache_pricing_columns"}, run: migrationAddFastModeCachePricingColumns}, {IDs: []string{"add_inference_geo_multiplier_column"}, run: migrationAddInferenceGeoMultiplierColumn}, + {IDs: []string{"add_flex_and_cache_creation_272k_pricing_columns"}, run: migrationAddFlexAndCacheCreation272kPricingColumns}, } // quoteSQLiteIdentifier quotes a SQLite identifier, escaping any double quotes. @@ -7875,6 +7876,49 @@ func migrationAddInferenceGeoMultiplierColumn(ctx context.Context, db *gorm.DB, return nil } +// migrationAddFlexAndCacheCreation272kPricingColumns adds the flex 272k-tier +// rates and the OpenAI cache-write (cache-creation) tiered rates introduced with +// gpt-5.6 (flex, priority, and the 272k context tier). +func migrationAddFlexAndCacheCreation272kPricingColumns(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "add_flex_and_cache_creation_272k_pricing_columns" + logger.Info("[configstore] starting migration %s", migrationName) + defer logger.Info("[configstore] finished migration %s", migrationName) + columns := []string{ + "input_cost_per_token_flex_above_272k_tokens", + "output_cost_per_token_flex_above_272k_tokens", + "cache_read_input_token_cost_flex_above_272k_tokens", + "cache_creation_input_token_cost_above_272k_tokens", + "cache_creation_input_token_cost_flex", + "cache_creation_input_token_cost_flex_above_272k_tokens", + "cache_creation_input_token_cost_priority", + } + m := migrator.New(db, migrator.DefaultOptions, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := addColumnIfNotExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to add column %s: %w", field, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, field := range columns { + if err := dropColumnIfExists(tx, logger, &tables.TableModelPricing{}, field); err != nil { + return fmt.Errorf("failed to drop column %s: %w", field, err) + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while running flex and cache creation 272k pricing columns migration: %s", err.Error()) + } + return nil +} + // migrationAddWhitelistedRoutesJSONColumn adds the whitelisted_routes_json column to the config_client table func migrationAddWhitelistedRoutesJSONColumn(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { migrationName := "add_whitelisted_routes_json_column" diff --git a/framework/configstore/rdb.go b/framework/configstore/rdb.go index 60e36069516..320e8551f09 100644 --- a/framework/configstore/rdb.go +++ b/framework/configstore/rdb.go @@ -2475,8 +2475,10 @@ var pricingSyncUpdateColumns = []string{ // Costs - 272k Tier "input_cost_per_token_above_272k_tokens", "input_cost_per_token_above_272k_tokens_priority", + "input_cost_per_token_flex_above_272k_tokens", "output_cost_per_token_above_272k_tokens", "output_cost_per_token_above_272k_tokens_priority", + "output_cost_per_token_flex_above_272k_tokens", // Costs - Cache "cache_creation_input_token_cost", "cache_read_input_token_cost", @@ -2491,6 +2493,11 @@ var pricingSyncUpdateColumns = []string{ "cache_read_input_image_token_cost", "cache_read_input_token_cost_above_272k_tokens", "cache_read_input_token_cost_above_272k_tokens_priority", + "cache_read_input_token_cost_flex_above_272k_tokens", + "cache_creation_input_token_cost_above_272k_tokens", + "cache_creation_input_token_cost_flex", + "cache_creation_input_token_cost_flex_above_272k_tokens", + "cache_creation_input_token_cost_priority", "cache_creation_input_token_cost_fast", "cache_creation_input_token_cost_above_1hr_fast", "cache_read_input_token_cost_fast", diff --git a/framework/configstore/tables/modelpricing.go b/framework/configstore/tables/modelpricing.go index 251bbf5014e..397d7e7afc1 100644 --- a/framework/configstore/tables/modelpricing.go +++ b/framework/configstore/tables/modelpricing.go @@ -48,8 +48,10 @@ type TableModelPricing struct { // Costs - 272k Tier InputCostPerTokenAbove272kTokens *float64 `gorm:"default:null;column:input_cost_per_token_above_272k_tokens" json:"input_cost_per_token_above_272k_tokens,omitempty"` InputCostPerTokenAbove272kTokensPriority *float64 `gorm:"default:null;column:input_cost_per_token_above_272k_tokens_priority" json:"input_cost_per_token_above_272k_tokens_priority,omitempty"` + InputCostPerTokenFlexAbove272kTokens *float64 `gorm:"default:null;column:input_cost_per_token_flex_above_272k_tokens" json:"input_cost_per_token_flex_above_272k_tokens,omitempty"` OutputCostPerTokenAbove272kTokens *float64 `gorm:"default:null;column:output_cost_per_token_above_272k_tokens" json:"output_cost_per_token_above_272k_tokens,omitempty"` OutputCostPerTokenAbove272kTokensPriority *float64 `gorm:"default:null;column:output_cost_per_token_above_272k_tokens_priority" json:"output_cost_per_token_above_272k_tokens_priority,omitempty"` + OutputCostPerTokenFlexAbove272kTokens *float64 `gorm:"default:null;column:output_cost_per_token_flex_above_272k_tokens" json:"output_cost_per_token_flex_above_272k_tokens,omitempty"` // Costs - Cache CacheCreationInputTokenCost *float64 `gorm:"default:null;column:cache_creation_input_token_cost" json:"cache_creation_input_token_cost,omitempty"` @@ -65,6 +67,12 @@ type TableModelPricing struct { CacheReadInputImageTokenCost *float64 `gorm:"default:null;column:cache_read_input_image_token_cost" json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens" json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `gorm:"default:null;column:cache_read_input_token_cost_above_272k_tokens_priority" json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + CacheReadInputTokenCostFlexAbove272kTokens *float64 `gorm:"default:null;column:cache_read_input_token_cost_flex_above_272k_tokens" json:"cache_read_input_token_cost_flex_above_272k_tokens,omitempty"` + // OpenAI cache-write (cache-creation) tiered rates, added with gpt-5.6. + CacheCreationInputTokenCostAbove272kTokens *float64 `gorm:"default:null;column:cache_creation_input_token_cost_above_272k_tokens" json:"cache_creation_input_token_cost_above_272k_tokens,omitempty"` + CacheCreationInputTokenCostFlex *float64 `gorm:"default:null;column:cache_creation_input_token_cost_flex" json:"cache_creation_input_token_cost_flex,omitempty"` + CacheCreationInputTokenCostFlexAbove272kTokens *float64 `gorm:"default:null;column:cache_creation_input_token_cost_flex_above_272k_tokens" json:"cache_creation_input_token_cost_flex_above_272k_tokens,omitempty"` + CacheCreationInputTokenCostPriority *float64 `gorm:"default:null;column:cache_creation_input_token_cost_priority" json:"cache_creation_input_token_cost_priority,omitempty"` // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. CacheCreationInputTokenCostFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_fast" json:"cache_creation_input_token_cost_fast,omitempty"` CacheCreationInputTokenCostAbove1hrFast *float64 `gorm:"default:null;column:cache_creation_input_token_cost_above_1hr_fast" json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` diff --git a/framework/modelcatalog/datasheet/cost.go b/framework/modelcatalog/datasheet/cost.go index 2394c87e44e..2411f0d9e6f 100644 --- a/framework/modelcatalog/datasheet/cost.go +++ b/framework/modelcatalog/datasheet/cost.go @@ -813,8 +813,13 @@ func tieredInputRate(pricing *configstoreTables.TableModelPricing, totalTokens i if tier.isFast && pricing.InputCostPerTokenFast != nil { return *pricing.InputCostPerTokenFast } - if tier.isFlex && pricing.InputCostPerTokenFlex != nil { - return *pricing.InputCostPerTokenFlex + if tier.isFlex { + if totalTokens > TokenTierAbove272K && pricing.InputCostPerTokenFlexAbove272kTokens != nil { + return *pricing.InputCostPerTokenFlexAbove272kTokens + } + if pricing.InputCostPerTokenFlex != nil { + return *pricing.InputCostPerTokenFlex + } } if totalTokens > TokenTierAbove272K { if tier.isPriority && pricing.InputCostPerTokenAbove272kTokensPriority != nil { @@ -852,8 +857,13 @@ func tieredOutputRate(pricing *configstoreTables.TableModelPricing, totalTokens if tier.isFast && pricing.OutputCostPerTokenFast != nil { return *pricing.OutputCostPerTokenFast } - if tier.isFlex && pricing.OutputCostPerTokenFlex != nil { - return *pricing.OutputCostPerTokenFlex + if tier.isFlex { + if totalTokens > TokenTierAbove272K && pricing.OutputCostPerTokenFlexAbove272kTokens != nil { + return *pricing.OutputCostPerTokenFlexAbove272kTokens + } + if pricing.OutputCostPerTokenFlex != nil { + return *pricing.OutputCostPerTokenFlex + } } if totalTokens > TokenTierAbove272K { if tier.isPriority && pricing.OutputCostPerTokenAbove272kTokensPriority != nil { @@ -955,8 +965,13 @@ func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, if tier.isFast && pricing.CacheReadInputTokenCostFast != nil { return *pricing.CacheReadInputTokenCostFast } - if tier.isFlex && pricing.CacheReadInputTokenCostFlex != nil { - return *pricing.CacheReadInputTokenCostFlex + if tier.isFlex { + if totalTokens > TokenTierAbove272K && pricing.CacheReadInputTokenCostFlexAbove272kTokens != nil { + return *pricing.CacheReadInputTokenCostFlexAbove272kTokens + } + if pricing.CacheReadInputTokenCostFlex != nil { + return *pricing.CacheReadInputTokenCostFlex + } } if totalTokens > TokenTierAbove272K { if tier.isPriority && pricing.CacheReadInputTokenCostAbove272kTokensPriority != nil { @@ -983,14 +998,32 @@ func tieredCacheReadInputTokenRate(pricing *configstoreTables.TableModelPricing, return tieredInputRate(pricing, totalTokens, tier) } -// Note: flex tier is not checked here because cache creation is not a concept in -// OpenAI's pricing model (the only provider that uses flex tier). Only cache read -// has a flex-specific rate. +// OpenAI introduced cache-write (cache-creation) pricing with gpt-5.6, tiered by +// service tier (flex/priority) and by the 272k context window; Anthropic uses the +// flat fast rate. Precedence mirrors tieredCacheReadInputTokenRate. func tieredCacheCreationInputTokenRate(pricing *configstoreTables.TableModelPricing, totalTokens int, tier serviceTier) float64 { // Fast mode (Anthropic) is a flat rate across the full context window. if tier.isFast && pricing.CacheCreationInputTokenCostFast != nil { return *pricing.CacheCreationInputTokenCostFast } + if tier.isFlex { + if totalTokens > TokenTierAbove272K && pricing.CacheCreationInputTokenCostFlexAbove272kTokens != nil { + return *pricing.CacheCreationInputTokenCostFlexAbove272kTokens + } + if pricing.CacheCreationInputTokenCostFlex != nil { + return *pricing.CacheCreationInputTokenCostFlex + } + } + // Priority has no long context: OpenAI does not offer priority >272k, and billing + // uses the served tier (response.service_tier), so an actual-priority request is + // always ≤272k. Its cache-write rate is flat, so it takes precedence over the + // standard context tiers below (which would otherwise capture the 200k–272k band). + if tier.isPriority && pricing.CacheCreationInputTokenCostPriority != nil { + return *pricing.CacheCreationInputTokenCostPriority + } + if totalTokens > TokenTierAbove272K && pricing.CacheCreationInputTokenCostAbove272kTokens != nil { + return *pricing.CacheCreationInputTokenCostAbove272kTokens + } if totalTokens > TokenTierAbove200K && pricing.CacheCreationInputTokenCostAbove200kTokens != nil { return *pricing.CacheCreationInputTokenCostAbove200kTokens } diff --git a/framework/modelcatalog/datasheet/cost_test.go b/framework/modelcatalog/datasheet/cost_test.go index 1892946874b..8e1518ebc72 100644 --- a/framework/modelcatalog/datasheet/cost_test.go +++ b/framework/modelcatalog/datasheet/cost_test.go @@ -1,6 +1,7 @@ package datasheet import ( + "encoding/json" "testing" bifrost "github.com/maximhq/bifrost/core" @@ -153,6 +154,126 @@ func TestComputeTextCost_WithCachedPromptTokens(t *testing.T) { assert.InDelta(t, 0.0096, cost, 1e-12) } +// gpt56SolPricing returns the full tiered pricing for gpt-5.6-sol (per the OpenAI +// pricing page), including flex/priority and the 272k context tier used to +// exercise the cache-write (cache-creation) tiering added with gpt-5.6. +func gpt56SolPricing() configstoreTables.TableModelPricing { + p := chatPricing(0.000005, 0.00003) // standard input $5/M, output $30/M + p.InputCostPerTokenAbove272kTokens = bifrost.Ptr(0.00001) + p.InputCostPerTokenFlex = bifrost.Ptr(0.0000025) + p.InputCostPerTokenFlexAbove272kTokens = bifrost.Ptr(0.000005) + p.InputCostPerTokenPriority = bifrost.Ptr(0.00001) + p.OutputCostPerTokenAbove272kTokens = bifrost.Ptr(0.000045) + p.OutputCostPerTokenFlex = bifrost.Ptr(0.000015) + p.OutputCostPerTokenFlexAbove272kTokens = bifrost.Ptr(0.0000225) + p.OutputCostPerTokenPriority = bifrost.Ptr(0.00006) + p.CacheReadInputTokenCost = bifrost.Ptr(0.0000005) + p.CacheReadInputTokenCostAbove272kTokens = bifrost.Ptr(0.000001) + p.CacheReadInputTokenCostFlex = bifrost.Ptr(0.00000025) + p.CacheReadInputTokenCostFlexAbove272kTokens = bifrost.Ptr(0.0000005) + p.CacheReadInputTokenCostPriority = bifrost.Ptr(0.000001) + p.CacheCreationInputTokenCost = bifrost.Ptr(0.00000625) + p.CacheCreationInputTokenCostAbove272kTokens = bifrost.Ptr(0.0000125) + p.CacheCreationInputTokenCostFlex = bifrost.Ptr(0.000003125) + p.CacheCreationInputTokenCostFlexAbove272kTokens = bifrost.Ptr(0.00000625) + p.CacheCreationInputTokenCostPriority = bifrost.Ptr(0.0000125) + return p +} + +// The reported bug scenario: a fresh-cache gpt-5.6-sol turn now returns and bills +// cache-write tokens at the base cache-creation rate. +func TestComputeTextCost_GPT56_StandardCacheWrite(t *testing.T) { + p := gpt56SolPricing() + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 28006, + CompletionTokens: 443, + TotalTokens: 28449, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 0, + CachedWriteTokens: 28003, + }, + } + cost := computeTextCost(&p, usage, serviceTier{}) + // input: (28006-28003)*0.000005 = 0.000015 + // write: 28003*0.00000625 = 0.17501875 + // output: 443*0.00003 = 0.01329 + assert.InDelta(t, 0.000015+0.17501875+0.01329, cost, 1e-9) +} + +func TestComputeTextCost_GPT56_StandardCacheWriteAbove272k(t *testing.T) { + p := gpt56SolPricing() + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 300000, + CompletionTokens: 1000, + TotalTokens: 301000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedWriteTokens: 100000}, + } + cost := computeTextCost(&p, usage, serviceTier{}) + // input: 200000*0.00001 = 2.0; write: 100000*0.0000125 = 1.25; output: 1000*0.000045 = 0.045 + assert.InDelta(t, 2.0+1.25+0.045, cost, 1e-9) +} + +func TestComputeTextCost_GPT56_FlexTier(t *testing.T) { + p := gpt56SolPricing() + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 10000, + CompletionTokens: 1000, + TotalTokens: 11000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 4000, + CachedWriteTokens: 2000, + }, + } + cost := computeTextCost(&p, usage, serviceTier{isFlex: true}) + // input: 4000*0.0000025 = 0.01; read: 4000*0.00000025 = 0.001 + // write: 2000*0.000003125 = 0.00625; output: 1000*0.000015 = 0.015 + assert.InDelta(t, 0.01+0.001+0.00625+0.015, cost, 1e-12) +} + +func TestComputeTextCost_GPT56_FlexTierAbove272k(t *testing.T) { + p := gpt56SolPricing() + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 300000, + CompletionTokens: 1000, + TotalTokens: 301000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 100000, + CachedWriteTokens: 50000, + }, + } + cost := computeTextCost(&p, usage, serviceTier{isFlex: true}) + // input: 150000*0.000005 = 0.75; read: 100000*0.0000005 = 0.05 + // write: 50000*0.00000625 = 0.3125; output: 1000*0.0000225 = 0.0225 + assert.InDelta(t, 0.75+0.05+0.3125+0.0225, cost, 1e-9) +} + +func TestComputeTextCost_GPT56_PriorityTier(t *testing.T) { + p := gpt56SolPricing() + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 10000, + CompletionTokens: 1000, + TotalTokens: 11000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 4000, + CachedWriteTokens: 2000, + }, + } + cost := computeTextCost(&p, usage, serviceTier{isPriority: true}) + // input: 4000*0.00001 = 0.04; read: 4000*0.000001 = 0.004 + // write: 2000*0.0000125 = 0.025; output: 1000*0.00006 = 0.06 + assert.InDelta(t, 0.04+0.004+0.025+0.06, cost, 1e-12) +} + +// Regression: a flex model without a flex-272k column keeps the flat flex rate +// above 272k (no new tiering leaks into existing flex models). +func TestComputeTextCost_FlexFlatAbove272kWhenNoFlexTierColumn(t *testing.T) { + p := chatPricing(0.000005, 0.00003) + p.InputCostPerTokenFlex = bifrost.Ptr(0.0000025) + usage := &schemas.BifrostLLMUsage{PromptTokens: 300000, TotalTokens: 300000} + cost := computeTextCost(&p, usage, serviceTier{isFlex: true}) + assert.InDelta(t, 300000*0.0000025, cost, 1e-9) +} + func TestComputeTextCost_FastMode(t *testing.T) { // Opus 4.8: standard $5/$25, fast $10/$50 per MTok. p := chatPricing(0.000005, 0.000025) @@ -3178,3 +3299,307 @@ func TestCalculateCostForUsage_AppliesServedTier(t *testing.T) { // inference_geo:"us" → 1.1x on base. assert.InDelta(t, (1000*0.000005+500*0.000015)*1.1, cost(mk(nil, &us)), 1e-12) } + +// =========================================================================== +// Golden per-model OpenAI pricing tests +// +// These feed the exact published OpenAI rates through the real datasheet +// JSON -> TableModelPricing conversion (convertEntryToTablePricing) and assert +// the invoice cost for every service tier and context window. They pin two +// things at once: the rate-selection ladders in cost.go, and the field wiring +// that carries each new rate from the datasheet JSON into the pricing row. +// +// NOTE: the numbers below are the source of truth for *expected billing*, +// transcribed from the OpenAI pricing page. The live per-model values arrive +// via the datasheet sync (not this repo), so these tests validate the compute +// engine + wiring, not the sync payload. +// =========================================================================== + +// pricingRowFromDatasheetJSON parses a datasheet entry (the shape stored in the +// synced pricing catalog) and runs the production JSON -> row conversion, so a +// dropped json tag or a missing conversion-map line would fail these tests. +func pricingRowFromDatasheetJSON(t *testing.T, modelKey, blob string) configstoreTables.TableModelPricing { + t.Helper() + var entry Entry + require.NoError(t, json.Unmarshal([]byte(blob), &entry)) + return convertEntryToTablePricing(modelKey, entry) +} + +// Datasheet entries (pricing fields only) for the gpt-5.6 flagship family. +// Cache writes are the cache_creation_* fields. Short/long context are the +// base vs _above_272k rates. +const ( + gpt56SolDatasheet = `{ + "provider": "openai", "mode": "chat", "base_model": "gpt-5.6-sol", + "input_cost_per_token": 0.000005, + "input_cost_per_token_above_272k_tokens": 0.00001, + "input_cost_per_token_flex": 0.0000025, + "input_cost_per_token_flex_above_272k_tokens": 0.000005, + "input_cost_per_token_priority": 0.00001, + "output_cost_per_token": 0.00003, + "output_cost_per_token_above_272k_tokens": 0.000045, + "output_cost_per_token_flex": 0.000015, + "output_cost_per_token_flex_above_272k_tokens": 0.0000225, + "output_cost_per_token_priority": 0.00006, + "cache_read_input_token_cost": 0.0000005, + "cache_read_input_token_cost_above_272k_tokens": 0.000001, + "cache_read_input_token_cost_flex": 0.00000025, + "cache_read_input_token_cost_flex_above_272k_tokens": 0.0000005, + "cache_read_input_token_cost_priority": 0.000001, + "cache_creation_input_token_cost": 0.00000625, + "cache_creation_input_token_cost_above_272k_tokens": 0.0000125, + "cache_creation_input_token_cost_flex": 0.000003125, + "cache_creation_input_token_cost_flex_above_272k_tokens": 0.00000625, + "cache_creation_input_token_cost_priority": 0.0000125 + }` + gpt56TerraDatasheet = `{ + "provider": "openai", "mode": "chat", "base_model": "gpt-5.6-terra", + "input_cost_per_token": 0.0000025, + "input_cost_per_token_above_272k_tokens": 0.000005, + "input_cost_per_token_flex": 0.00000125, + "input_cost_per_token_flex_above_272k_tokens": 0.0000025, + "input_cost_per_token_priority": 0.000005, + "output_cost_per_token": 0.000015, + "output_cost_per_token_above_272k_tokens": 0.0000225, + "output_cost_per_token_flex": 0.0000075, + "output_cost_per_token_flex_above_272k_tokens": 0.00001125, + "output_cost_per_token_priority": 0.00003, + "cache_read_input_token_cost": 0.00000025, + "cache_read_input_token_cost_above_272k_tokens": 0.0000005, + "cache_read_input_token_cost_flex": 0.000000125, + "cache_read_input_token_cost_flex_above_272k_tokens": 0.00000025, + "cache_read_input_token_cost_priority": 0.0000005, + "cache_creation_input_token_cost": 0.000003125, + "cache_creation_input_token_cost_above_272k_tokens": 0.00000625, + "cache_creation_input_token_cost_flex": 0.0000015625, + "cache_creation_input_token_cost_flex_above_272k_tokens": 0.000003125, + "cache_creation_input_token_cost_priority": 0.00000625 + }` + gpt56LunaDatasheet = `{ + "provider": "openai", "mode": "chat", "base_model": "gpt-5.6-luna", + "input_cost_per_token": 0.000001, + "input_cost_per_token_above_272k_tokens": 0.000002, + "input_cost_per_token_flex": 0.0000005, + "input_cost_per_token_flex_above_272k_tokens": 0.000001, + "input_cost_per_token_priority": 0.000002, + "output_cost_per_token": 0.000006, + "output_cost_per_token_above_272k_tokens": 0.000009, + "output_cost_per_token_flex": 0.000003, + "output_cost_per_token_flex_above_272k_tokens": 0.0000045, + "output_cost_per_token_priority": 0.000012, + "cache_read_input_token_cost": 0.0000001, + "cache_read_input_token_cost_above_272k_tokens": 0.0000002, + "cache_read_input_token_cost_flex": 0.00000005, + "cache_read_input_token_cost_flex_above_272k_tokens": 0.0000001, + "cache_read_input_token_cost_priority": 0.0000002, + "cache_creation_input_token_cost": 0.00000125, + "cache_creation_input_token_cost_above_272k_tokens": 0.0000025, + "cache_creation_input_token_cost_flex": 0.000000625, + "cache_creation_input_token_cost_flex_above_272k_tokens": 0.00000125, + "cache_creation_input_token_cost_priority": 0.0000025 + }` + // gpt-5.5: long-context tiering but NO published cache-write rate ("-"). + gpt55Datasheet = `{ + "provider": "openai", "mode": "chat", "base_model": "gpt-5.5", + "input_cost_per_token": 0.000005, + "input_cost_per_token_above_272k_tokens": 0.00001, + "input_cost_per_token_flex": 0.0000025, + "input_cost_per_token_flex_above_272k_tokens": 0.000005, + "input_cost_per_token_priority": 0.0000125, + "output_cost_per_token": 0.00003, + "output_cost_per_token_above_272k_tokens": 0.000045, + "output_cost_per_token_flex": 0.000015, + "output_cost_per_token_flex_above_272k_tokens": 0.0000225, + "output_cost_per_token_priority": 0.000075, + "cache_read_input_token_cost": 0.0000005, + "cache_read_input_token_cost_above_272k_tokens": 0.000001, + "cache_read_input_token_cost_flex": 0.00000025, + "cache_read_input_token_cost_flex_above_272k_tokens": 0.0000005, + "cache_read_input_token_cost_priority": 0.00000125 + }` + // gpt-5.4-mini: NO long-context tier and NO cache-write rate. + gpt54MiniDatasheet = `{ + "provider": "openai", "mode": "chat", "base_model": "gpt-5.4-mini", + "input_cost_per_token": 0.00000075, + "input_cost_per_token_flex": 0.000000375, + "input_cost_per_token_priority": 0.0000015, + "output_cost_per_token": 0.0000045, + "output_cost_per_token_flex": 0.00000225, + "output_cost_per_token_priority": 0.000009, + "cache_read_input_token_cost": 0.000000075, + "cache_read_input_token_cost_flex": 0.0000000375, + "cache_read_input_token_cost_priority": 0.00000015 + }` +) + +// TestGoldenOpenAIPricing_GPT56Family asserts the exact invoice cost for every +// (tier x context) cell of the gpt-5.6 pricing tables, including cache read and +// cache write. Each row hardcodes the rate that *should* apply, so a mis-selected +// tier or context bucket fails the assertion. +func TestGoldenOpenAIPricing_GPT56Family(t *testing.T) { + sol := pricingRowFromDatasheetJSON(t, "gpt-5.6-sol", gpt56SolDatasheet) + terra := pricingRowFromDatasheetJSON(t, "gpt-5.6-terra", gpt56TerraDatasheet) + luna := pricingRowFromDatasheetJSON(t, "gpt-5.6-luna", gpt56LunaDatasheet) + + // Short context stays under 272k; long context crosses it. + const ( + shortPrompt, shortRead, shortWrite, shortOut = 10000, 4000, 2000, 1000 + longPrompt, longRead, longWrite, longOut = 300000, 100000, 50000, 1000 + ) + + type rates struct{ in, cacheRead, cacheWrite, out float64 } + cases := []struct { + name string + pricing configstoreTables.TableModelPricing + tier serviceTier + long bool + r rates + }{ + // gpt-5.6-sol + {"sol/standard/short", sol, serviceTier{}, false, rates{0.000005, 0.0000005, 0.00000625, 0.00003}}, + {"sol/standard/long", sol, serviceTier{}, true, rates{0.00001, 0.000001, 0.0000125, 0.000045}}, + {"sol/flex/short", sol, serviceTier{isFlex: true}, false, rates{0.0000025, 0.00000025, 0.000003125, 0.000015}}, + {"sol/flex/long", sol, serviceTier{isFlex: true}, true, rates{0.000005, 0.0000005, 0.00000625, 0.0000225}}, + {"sol/priority/short", sol, serviceTier{isPriority: true}, false, rates{0.00001, 0.000001, 0.0000125, 0.00006}}, + // gpt-5.6-terra + {"terra/standard/short", terra, serviceTier{}, false, rates{0.0000025, 0.00000025, 0.000003125, 0.000015}}, + {"terra/standard/long", terra, serviceTier{}, true, rates{0.000005, 0.0000005, 0.00000625, 0.0000225}}, + {"terra/flex/short", terra, serviceTier{isFlex: true}, false, rates{0.00000125, 0.000000125, 0.0000015625, 0.0000075}}, + {"terra/flex/long", terra, serviceTier{isFlex: true}, true, rates{0.0000025, 0.00000025, 0.000003125, 0.00001125}}, + {"terra/priority/short", terra, serviceTier{isPriority: true}, false, rates{0.000005, 0.0000005, 0.00000625, 0.00003}}, + // gpt-5.6-luna + {"luna/standard/short", luna, serviceTier{}, false, rates{0.000001, 0.0000001, 0.00000125, 0.000006}}, + {"luna/standard/long", luna, serviceTier{}, true, rates{0.000002, 0.0000002, 0.0000025, 0.000009}}, + {"luna/flex/short", luna, serviceTier{isFlex: true}, false, rates{0.0000005, 0.00000005, 0.000000625, 0.000003}}, + {"luna/flex/long", luna, serviceTier{isFlex: true}, true, rates{0.000001, 0.0000001, 0.00000125, 0.0000045}}, + {"luna/priority/short", luna, serviceTier{isPriority: true}, false, rates{0.000002, 0.0000002, 0.0000025, 0.000012}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + prompt, read, write, out := shortPrompt, shortRead, shortWrite, shortOut + if tc.long { + prompt, read, write, out = longPrompt, longRead, longWrite, longOut + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: prompt, + CompletionTokens: out, + TotalTokens: prompt + out, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: read, + CachedWriteTokens: write, + }, + } + p := tc.pricing + cost := computeTextCost(&p, usage, tc.tier) + nonCached := prompt - read - write + want := float64(nonCached)*tc.r.in + float64(read)*tc.r.cacheRead + float64(write)*tc.r.cacheWrite + float64(out)*tc.r.out + assert.InDelta(t, want, cost, 1e-9) + }) + } +} + +// TestGoldenOpenAIPricing_NoCacheWriteModels covers models that OpenAI prices +// without a cache-write rate (gpt-5.5) and without a long-context tier +// (gpt-5.4-mini): cache-write tokens fall back to the input rate, and a >272k +// request on a model with no long-context rate stays on the base rate. +func TestGoldenOpenAIPricing_NoCacheWriteModels(t *testing.T) { + gpt55 := pricingRowFromDatasheetJSON(t, "gpt-5.5", gpt55Datasheet) + gpt54mini := pricingRowFromDatasheetJSON(t, "gpt-5.4-mini", gpt54MiniDatasheet) + + // gpt-5.5 has no cache-write rate: write tokens bill at the (long-context) input rate. + t.Run("gpt-5.5/standard/long: cache-write falls back to input rate", func(t *testing.T) { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 300000, CompletionTokens: 1000, TotalTokens: 301000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedReadTokens: 100000, CachedWriteTokens: 50000}, + } + p := gpt55 + cost := computeTextCost(&p, usage, serviceTier{}) + // non-cached 150000*0.00001 + read 100000*0.000001 + write 50000*0.00001 (input fallback) + out 1000*0.000045 + want := 150000*0.00001 + 100000*0.000001 + 50000*0.00001 + 1000*0.000045 + assert.InDelta(t, want, cost, 1e-9) + }) + + t.Run("gpt-5.5/flex/short", func(t *testing.T) { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 10000, CompletionTokens: 1000, TotalTokens: 11000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedReadTokens: 4000}, + } + p := gpt55 + cost := computeTextCost(&p, usage, serviceTier{isFlex: true}) + want := 6000*0.0000025 + 4000*0.00000025 + 1000*0.000015 + assert.InDelta(t, want, cost, 1e-12) + }) + + t.Run("gpt-5.5/priority/short", func(t *testing.T) { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 10000, CompletionTokens: 1000, TotalTokens: 11000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedReadTokens: 4000}, + } + p := gpt55 + cost := computeTextCost(&p, usage, serviceTier{isPriority: true}) + want := 6000*0.0000125 + 4000*0.00000125 + 1000*0.000075 + assert.InDelta(t, want, cost, 1e-9) + }) + + // gpt-5.4-mini has no long-context rate: a >272k request must use the base rate. + t.Run("gpt-5.4-mini/standard/above-272k uses base rate", func(t *testing.T) { + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 300000, CompletionTokens: 1000, TotalTokens: 301000, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedReadTokens: 50000}, + } + p := gpt54mini + cost := computeTextCost(&p, usage, serviceTier{}) + want := 250000*0.00000075 + 50000*0.000000075 + 1000*0.0000045 + assert.InDelta(t, want, cost, 1e-9) + }) +} + +// TestCalculateCost_GPT56_Responses_FlexLongContext_EndToEnd drives the full +// pipeline for a gpt-5.6 Responses call: service_tier=flex + a >272k prompt with +// cache writes, through tier detection, usage mapping, pricing lookup, and cost. +func TestCalculateCost_GPT56_Responses_FlexLongContext_EndToEnd(t *testing.T) { + sol := pricingRowFromDatasheetJSON(t, "gpt-5.6-sol", gpt56SolDatasheet) + s := testStoreWithPricing(map[string]configstoreTables.TableModelPricing{ + makeKey("gpt-5.6-sol", "openai", "responses"): sol, + }) + tier := schemas.BifrostServiceTierFlex + resp := &schemas.BifrostResponse{ + ResponsesResponse: &schemas.BifrostResponsesResponse{ + ServiceTier: &tier, + Usage: &schemas.ResponsesResponseUsage{ + InputTokens: 300000, + OutputTokens: 1000, + TotalTokens: 301000, + InputTokensDetails: &schemas.ResponsesResponseInputTokens{ + CachedReadTokens: 100000, + CachedWriteTokens: 50000, + }, + }, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ResponsesRequest, + RoutingInfo: routingInfoFor(schemas.OpenAI, "gpt-5.6-sol"), + }, + }, + } + cost := s.CalculateCost(resp, nil) + // flex long-context: 150000*0.000005 + 100000*0.0000005 + 50000*0.00000625 + 1000*0.0000225 + want := 150000*0.000005 + 100000*0.0000005 + 50000*0.00000625 + 1000*0.0000225 + assert.InDelta(t, want, cost, 1e-9) +} + +// TestTieredCacheCreationRate_PriorityWinsOver200kBand verifies a priority cache-write +// request in the 200k–272k band uses the flat priority rate, not the standard >200k +// rate. Priority has no long context (OpenAI does not offer priority >272k), so its +// flat rate must take precedence over the standard context tiers. +func TestTieredCacheCreationRate_PriorityWinsOver200kBand(t *testing.T) { + p := configstoreTables.TableModelPricing{ + CacheCreationInputTokenCost: bifrost.Ptr(0.000001), + CacheCreationInputTokenCostAbove200kTokens: bifrost.Ptr(0.000002), // standard >200k — must NOT win for priority + CacheCreationInputTokenCostPriority: bifrost.Ptr(0.000005), // flat priority — must win + } + // 250k tokens: >200k and ≤272k. + assert.Equal(t, 0.000005, tieredCacheCreationInputTokenRate(&p, 250000, serviceTier{isPriority: true})) + // Non-priority at the same size still uses the standard >200k rate. + assert.Equal(t, 0.000002, tieredCacheCreationInputTokenRate(&p, 250000, serviceTier{})) +} diff --git a/framework/modelcatalog/datasheet/overrides.go b/framework/modelcatalog/datasheet/overrides.go index 72151224d6f..1b805325428 100644 --- a/framework/modelcatalog/datasheet/overrides.go +++ b/framework/modelcatalog/datasheet/overrides.go @@ -275,8 +275,10 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options) {dst: &patched.OutputCostPerTokenAbove200kTokensPriority, src: override.OutputCostPerTokenAbove200kTokensPriority}, {dst: &patched.InputCostPerTokenAbove272kTokens, src: override.InputCostPerTokenAbove272kTokens}, {dst: &patched.InputCostPerTokenAbove272kTokensPriority, src: override.InputCostPerTokenAbove272kTokensPriority}, + {dst: &patched.InputCostPerTokenFlexAbove272kTokens, src: override.InputCostPerTokenFlexAbove272kTokens}, {dst: &patched.OutputCostPerTokenAbove272kTokens, src: override.OutputCostPerTokenAbove272kTokens}, {dst: &patched.OutputCostPerTokenAbove272kTokensPriority, src: override.OutputCostPerTokenAbove272kTokensPriority}, + {dst: &patched.OutputCostPerTokenFlexAbove272kTokens, src: override.OutputCostPerTokenFlexAbove272kTokens}, {dst: &patched.CacheCreationInputTokenCostAbove200kTokens, src: override.CacheCreationInputTokenCostAbove200kTokens}, {dst: &patched.CacheReadInputTokenCostAbove200kTokens, src: override.CacheReadInputTokenCostAbove200kTokens}, {dst: &patched.CacheReadInputTokenCost, src: override.CacheReadInputTokenCost}, @@ -289,6 +291,11 @@ func patchPricing(pricing configstoreTables.TableModelPricing, override Options) {dst: &patched.CacheReadInputTokenCostAbove200kTokensPriority, src: override.CacheReadInputTokenCostAbove200kTokensPriority}, {dst: &patched.CacheReadInputTokenCostAbove272kTokens, src: override.CacheReadInputTokenCostAbove272kTokens}, {dst: &patched.CacheReadInputTokenCostAbove272kTokensPriority, src: override.CacheReadInputTokenCostAbove272kTokensPriority}, + {dst: &patched.CacheReadInputTokenCostFlexAbove272kTokens, src: override.CacheReadInputTokenCostFlexAbove272kTokens}, + {dst: &patched.CacheCreationInputTokenCostAbove272kTokens, src: override.CacheCreationInputTokenCostAbove272kTokens}, + {dst: &patched.CacheCreationInputTokenCostFlex, src: override.CacheCreationInputTokenCostFlex}, + {dst: &patched.CacheCreationInputTokenCostFlexAbove272kTokens, src: override.CacheCreationInputTokenCostFlexAbove272kTokens}, + {dst: &patched.CacheCreationInputTokenCostPriority, src: override.CacheCreationInputTokenCostPriority}, {dst: &patched.CacheCreationInputTokenCostFast, src: override.CacheCreationInputTokenCostFast}, {dst: &patched.CacheCreationInputTokenCostAbove1hrFast, src: override.CacheCreationInputTokenCostAbove1hrFast}, {dst: &patched.CacheReadInputTokenCostFast, src: override.CacheReadInputTokenCostFast}, diff --git a/framework/modelcatalog/datasheet/types.go b/framework/modelcatalog/datasheet/types.go index 9078d64a776..da756b5ebc2 100644 --- a/framework/modelcatalog/datasheet/types.go +++ b/framework/modelcatalog/datasheet/types.go @@ -122,8 +122,10 @@ type Options struct { // Costs - 272k Tier InputCostPerTokenAbove272kTokens *float64 `json:"input_cost_per_token_above_272k_tokens,omitempty"` InputCostPerTokenAbove272kTokensPriority *float64 `json:"input_cost_per_token_above_272k_tokens_priority,omitempty"` + InputCostPerTokenFlexAbove272kTokens *float64 `json:"input_cost_per_token_flex_above_272k_tokens,omitempty"` OutputCostPerTokenAbove272kTokens *float64 `json:"output_cost_per_token_above_272k_tokens,omitempty"` OutputCostPerTokenAbove272kTokensPriority *float64 `json:"output_cost_per_token_above_272k_tokens_priority,omitempty"` + OutputCostPerTokenFlexAbove272kTokens *float64 `json:"output_cost_per_token_flex_above_272k_tokens,omitempty"` // Costs - Cache CacheCreationInputTokenCost *float64 `json:"cache_creation_input_token_cost,omitempty"` @@ -139,6 +141,12 @@ type Options struct { CacheReadInputImageTokenCost *float64 `json:"cache_read_input_image_token_cost,omitempty"` CacheReadInputTokenCostAbove272kTokens *float64 `json:"cache_read_input_token_cost_above_272k_tokens,omitempty"` CacheReadInputTokenCostAbove272kTokensPriority *float64 `json:"cache_read_input_token_cost_above_272k_tokens_priority,omitempty"` + CacheReadInputTokenCostFlexAbove272kTokens *float64 `json:"cache_read_input_token_cost_flex_above_272k_tokens,omitempty"` + // OpenAI cache-write (cache-creation) tiered rates, added with gpt-5.6. + CacheCreationInputTokenCostAbove272kTokens *float64 `json:"cache_creation_input_token_cost_above_272k_tokens,omitempty"` + CacheCreationInputTokenCostFlex *float64 `json:"cache_creation_input_token_cost_flex,omitempty"` + CacheCreationInputTokenCostFlexAbove272kTokens *float64 `json:"cache_creation_input_token_cost_flex_above_272k_tokens,omitempty"` + CacheCreationInputTokenCostPriority *float64 `json:"cache_creation_input_token_cost_priority,omitempty"` // Fast mode (Anthropic) cache rates — flat across the full context window, no tiering. CacheCreationInputTokenCostFast *float64 `json:"cache_creation_input_token_cost_fast,omitempty"` CacheCreationInputTokenCostAbove1hrFast *float64 `json:"cache_creation_input_token_cost_above_1hr_fast,omitempty"` @@ -578,8 +586,10 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables. OutputCostPerTokenAbove200kTokensPriority: entry.OutputCostPerTokenAbove200kTokensPriority, InputCostPerTokenAbove272kTokens: entry.InputCostPerTokenAbove272kTokens, InputCostPerTokenAbove272kTokensPriority: entry.InputCostPerTokenAbove272kTokensPriority, + InputCostPerTokenFlexAbove272kTokens: entry.InputCostPerTokenFlexAbove272kTokens, OutputCostPerTokenAbove272kTokens: entry.OutputCostPerTokenAbove272kTokens, OutputCostPerTokenAbove272kTokensPriority: entry.OutputCostPerTokenAbove272kTokensPriority, + OutputCostPerTokenFlexAbove272kTokens: entry.OutputCostPerTokenFlexAbove272kTokens, InputCostPerCharacter: entry.InputCostPerCharacter, InputCostPerTokenAbove128kTokens: entry.InputCostPerTokenAbove128kTokens, InputCostPerImageAbove128kTokens: entry.InputCostPerImageAbove128kTokens, @@ -600,6 +610,11 @@ func convertEntryToTablePricing(modelKey string, entry Entry) configstoreTables. CacheReadInputImageTokenCost: entry.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: entry.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: entry.CacheReadInputTokenCostAbove272kTokensPriority, + CacheReadInputTokenCostFlexAbove272kTokens: entry.CacheReadInputTokenCostFlexAbove272kTokens, + CacheCreationInputTokenCostAbove272kTokens: entry.CacheCreationInputTokenCostAbove272kTokens, + CacheCreationInputTokenCostFlex: entry.CacheCreationInputTokenCostFlex, + CacheCreationInputTokenCostFlexAbove272kTokens: entry.CacheCreationInputTokenCostFlexAbove272kTokens, + CacheCreationInputTokenCostPriority: entry.CacheCreationInputTokenCostPriority, CacheCreationInputTokenCostFast: entry.CacheCreationInputTokenCostFast, CacheCreationInputTokenCostAbove1hrFast: entry.CacheCreationInputTokenCostAbove1hrFast, CacheReadInputTokenCostFast: entry.CacheReadInputTokenCostFast, @@ -659,8 +674,10 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E OutputCostPerTokenAbove200kTokensPriority: pricing.OutputCostPerTokenAbove200kTokensPriority, InputCostPerTokenAbove272kTokens: pricing.InputCostPerTokenAbove272kTokens, InputCostPerTokenAbove272kTokensPriority: pricing.InputCostPerTokenAbove272kTokensPriority, + InputCostPerTokenFlexAbove272kTokens: pricing.InputCostPerTokenFlexAbove272kTokens, OutputCostPerTokenAbove272kTokens: pricing.OutputCostPerTokenAbove272kTokens, OutputCostPerTokenAbove272kTokensPriority: pricing.OutputCostPerTokenAbove272kTokensPriority, + OutputCostPerTokenFlexAbove272kTokens: pricing.OutputCostPerTokenFlexAbove272kTokens, InputCostPerCharacter: pricing.InputCostPerCharacter, InputCostPerTokenAbove128kTokens: pricing.InputCostPerTokenAbove128kTokens, InputCostPerImageAbove128kTokens: pricing.InputCostPerImageAbove128kTokens, @@ -681,6 +698,11 @@ func convertTablePricingToEntry(pricing *configstoreTables.TableModelPricing) *E CacheReadInputImageTokenCost: pricing.CacheReadInputImageTokenCost, CacheReadInputTokenCostAbove272kTokens: pricing.CacheReadInputTokenCostAbove272kTokens, CacheReadInputTokenCostAbove272kTokensPriority: pricing.CacheReadInputTokenCostAbove272kTokensPriority, + CacheReadInputTokenCostFlexAbove272kTokens: pricing.CacheReadInputTokenCostFlexAbove272kTokens, + CacheCreationInputTokenCostAbove272kTokens: pricing.CacheCreationInputTokenCostAbove272kTokens, + CacheCreationInputTokenCostFlex: pricing.CacheCreationInputTokenCostFlex, + CacheCreationInputTokenCostFlexAbove272kTokens: pricing.CacheCreationInputTokenCostFlexAbove272kTokens, + CacheCreationInputTokenCostPriority: pricing.CacheCreationInputTokenCostPriority, CacheCreationInputTokenCostFast: pricing.CacheCreationInputTokenCostFast, CacheCreationInputTokenCostAbove1hrFast: pricing.CacheCreationInputTokenCostAbove1hrFast, CacheReadInputTokenCostFast: pricing.CacheReadInputTokenCostFast, diff --git a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx index 1cb147af6b6..2b07fb4b6bc 100644 --- a/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx +++ b/ui/app/workspace/custom-pricing/overrides/pricingOverrideSheet.tsx @@ -152,6 +152,18 @@ export const PRICING_FIELDS = [ group: "chat", requestTypeGroups: ["chat"], }, + { + key: "input_cost_per_token_flex_above_272k_tokens", + label: "Input / token (>272k, flex)", + group: "chat", + requestTypeGroups: ["chat"], + }, + { + key: "output_cost_per_token_flex_above_272k_tokens", + label: "Output / token (>272k, flex)", + group: "chat", + requestTypeGroups: ["chat"], + }, { key: "cache_creation_input_token_cost", label: "Cache creation / token", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost", label: "Cache read / token", group: "chat", requestTypeGroups: ["chat"] }, { @@ -171,7 +183,12 @@ export const PRICING_FIELDS = [ { key: "cache_read_input_token_cost_priority", label: "Cache read / token (priority)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_flex", label: "Cache read / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_creation_input_token_cost_fast", label: "Cache creation / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, - { key: "cache_creation_input_token_cost_above_1hr_fast", label: "Cache creation / token (>1hr, fast)", group: "chat", requestTypeGroups: ["chat"] }, + { + key: "cache_creation_input_token_cost_above_1hr_fast", + label: "Cache creation / token (>1hr, fast)", + group: "chat", + requestTypeGroups: ["chat"], + }, { key: "cache_read_input_token_cost_fast", label: "Cache read / token (fast)", group: "chat", requestTypeGroups: ["chat"] }, { key: "cache_read_input_token_cost_above_200k_tokens_priority", @@ -186,6 +203,31 @@ export const PRICING_FIELDS = [ group: "chat", requestTypeGroups: ["chat"], }, + { + key: "cache_read_input_token_cost_flex_above_272k_tokens", + label: "Cache read / token (>272k, flex)", + group: "chat", + requestTypeGroups: ["chat"], + }, + { + key: "cache_creation_input_token_cost_priority", + label: "Cache creation / token (priority)", + group: "chat", + requestTypeGroups: ["chat"], + }, + { key: "cache_creation_input_token_cost_flex", label: "Cache creation / token (flex)", group: "chat", requestTypeGroups: ["chat"] }, + { + key: "cache_creation_input_token_cost_above_272k_tokens", + label: "Cache creation / token (>272k)", + group: "chat", + requestTypeGroups: ["chat"], + }, + { + key: "cache_creation_input_token_cost_flex_above_272k_tokens", + label: "Cache creation / token (>272k, flex)", + group: "chat", + requestTypeGroups: ["chat"], + }, { key: "search_context_cost_per_query", label: "Search context / query", group: "chat", requestTypeGroups: ["chat", "rerank"] }, { key: "code_interpreter_cost_per_session", label: "Code interpreter / session", group: "chat", requestTypeGroups: ["chat"] }, { key: "inference_geo_us_multiplier", label: "Inference geo US multiplier", group: "chat", requestTypeGroups: ["chat"] }, diff --git a/ui/lib/types/governance.ts b/ui/lib/types/governance.ts index 971fde9c24e..8be75135d42 100644 --- a/ui/lib/types/governance.ts +++ b/ui/lib/types/governance.ts @@ -439,8 +439,10 @@ export interface PricingOverridePatch { // 272k tier input_cost_per_token_above_272k_tokens?: number; input_cost_per_token_above_272k_tokens_priority?: number; + input_cost_per_token_flex_above_272k_tokens?: number; output_cost_per_token_above_272k_tokens?: number; output_cost_per_token_above_272k_tokens_priority?: number; + output_cost_per_token_flex_above_272k_tokens?: number; // Cache cache_creation_input_token_cost?: number; cache_read_input_token_cost?: number; @@ -455,6 +457,11 @@ export interface PricingOverridePatch { cache_read_input_image_token_cost?: number; cache_read_input_token_cost_above_272k_tokens?: number; cache_read_input_token_cost_above_272k_tokens_priority?: number; + cache_read_input_token_cost_flex_above_272k_tokens?: number; + cache_creation_input_token_cost_above_272k_tokens?: number; + cache_creation_input_token_cost_flex?: number; + cache_creation_input_token_cost_flex_above_272k_tokens?: number; + cache_creation_input_token_cost_priority?: number; cache_creation_input_token_cost_fast?: number; cache_creation_input_token_cost_above_1hr_fast?: number; cache_read_input_token_cost_fast?: number; From 7176f83319b9360ce0c3c4fb1803ef3a09f414bd Mon Sep 17 00:00:00 2001 From: tejas ghatte Date: Fri, 10 Jul 2026 12:47:46 +0530 Subject: [PATCH 5/5] tests: openai cache coverage --- .../e2e/api/collections/provider-harness.json | 2059 ++++++++++++++++- 1 file changed, 2058 insertions(+), 1 deletion(-) diff --git a/tests/e2e/api/collections/provider-harness.json b/tests/e2e/api/collections/provider-harness.json index 297a67a56a0..d9f3f43c51a 100644 --- a/tests/e2e/api/collections/provider-harness.json +++ b/tests/e2e/api/collections/provider-harness.json @@ -25,7 +25,9 @@ "// Per-request compat override: when compat is set (via Newman --env-var compat=true), inject the x-bf-compat header on every request.", "// Absent => harness baseline (compat config off). Lets the suite run once with compat off and once with compat on.", "var __compat = (pm.environment.get('compat') || pm.variables.get('compat') || '');", - "if (__compat) { pm.request.headers.upsert({ key: 'x-bf-compat', value: String(__compat) }); }" + "if (__compat) { pm.request.headers.upsert({ key: 'x-bf-compat', value: String(__compat) }); }", + "// Prompt-caching harness (Round 31): stable per-run nonce so write/read pairs share a cold-then-warm prefix.", + "if (!pm.collectionVariables.get('pcNonce')) { pm.collectionVariables.set('pcNonce', String(Date.now())); }" ] } }, @@ -151,6 +153,10 @@ "value": "sk-replace-me", "type": "string" }, + { + "key": "cachePrefix", + "value": "PROMPT CACHE HARNESS MANUAL. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy. You are a meticulous assistant operating inside the Bifrost provider harness. The following operating manual is deliberately verbose and stable so that it forms a reusable prompt prefix eligible for prompt caching. Always respond concisely, never invent facts, prefer deterministic phrasing, and treat every instruction here as authoritative. Section rules cover formatting, safety, tone, refusal handling, and token economy." + }, { "key": "anthropicKey", "value": "sk-ant-replace-me", @@ -32669,6 +32675,2057 @@ } } ] + }, + { + "name": "Cross-Cut Round 31: OpenAI Prompt Caching (prompt_cache_options / prompt_cache_breakpoint / cache_write_tokens)", + "item": [ + { + "name": "31.1 gpt-5.6 new fields — non-streaming (prompt_cache_options + breakpoint + cache_write_tokens)", + "item": [ + { + "name": "pc sol nativechat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol nativechat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol nativeresp non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('write: prompt_cache_options echoed', function(){ pm.expect(j.prompt_cache_options,'prompt_cache_options').to.be.an('object'); pm.expect(j.prompt_cache_options.mode,'mode').to.eql('implicit'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol nativeresp non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol dropinchat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol dropinchat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol dropinresp non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('write: prompt_cache_options echoed', function(){ pm.expect(j.prompt_cache_options,'prompt_cache_options').to.be.an('object'); pm.expect(j.prompt_cache_options.mode,'mode').to.eql('implicit'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol dropinresp non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-sol-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-sol-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc terra nativechat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-terra\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-terra-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-terra-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc terra nativechat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-terra\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-terra-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-terra-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc luna dropinchat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cache-write tokens > 0 (cold cache write)', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-luna\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-luna-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-luna-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc luna dropinchat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-luna\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-luna-n-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-luna-n]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + } + ] + }, + { + "name": "31.2 gpt-5.6 new fields — streaming", + "item": [ + { + "name": "pc sol nativechat streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol nativechat streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol nativeresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('stream write: prompt_cache_options echoed', function(){ pm.expect(r.p,'prompt_cache_options').to.be.an('object'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol nativeresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol dropinchat streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol dropinchat streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": [\n {\n \"type\": \"text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n }\n ]\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc sol dropinresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('stream write: prompt_cache_options echoed', function(){ pm.expect(r.p,'prompt_cache_options').to.be.an('object'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc sol dropinresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-sol\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-sol-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-sol-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc terra dropinresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('stream write: prompt_cache_options echoed', function(){ pm.expect(r.p,'prompt_cache_options').to.be.an('object'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-terra\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-terra-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-terra-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc terra dropinresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5.6-terra\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-terra-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-terra-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc luna nativeresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cache-write tokens > 0', function(){ pm.expect(wtok(d), 'cache write tokens').to.be.above(0); });", + " pm.test('stream write: prompt_cache_options echoed', function(){ pm.expect(r.p,'prompt_cache_options').to.be.an('object'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-luna\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-luna-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-luna-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc luna nativeresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5.6-luna\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-luna-s-{{pcNonce}}\",\n \"prompt_cache_options\": {\n \"mode\": \"implicit\",\n \"ttl\": \"30m\"\n },\n \"input\": [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_text\",\n \"text\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-luna-s]\",\n \"prompt_cache_breakpoint\": {\n \"mode\": \"explicit\"\n }\n },\n {\n \"type\": \"input_text\",\n \"text\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n }\n ],\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "31.3 legacy automatic caching — non-streaming (cached_tokens shape + cache hit)", + "item": [ + { + "name": "pc gpt-5 nativechat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-gpt-5-n-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-gpt-5-n]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-5 nativechat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-gpt-5-n-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-gpt-5-n]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-5-mini nativeresp non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5-mini\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-gpt-5-mini-n-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-gpt-5-mini-n] Answer in one short word: is the sky blue?\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-5-mini nativeresp non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-5-mini\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-gpt-5-mini-n-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-gpt-5-mini-n] Answer in one short word: is the sky blue?\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-5-nano dropinchat non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5-nano\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-gpt-5-nano-n-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-gpt-5-nano-n]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-5-nano dropinchat non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-5-nano\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-gpt-5-nano-n-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-gpt-5-nano-n]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ]\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-4.1 dropinresp non-streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('write: usage present', function(){ pm.expect(j.usage,'usage').to.be.an('object'); });", + " pm.test('write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4.1\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-gpt-4.1-n-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-gpt-4.1-n] Answer in one short word: is the sky blue?\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-4.1 dropinresp non-streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var j = pm.response.json();", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var d = det(j.usage);", + " pm.test('read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4.1\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-gpt-4.1-n-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-gpt-4.1-n] Answer in one short word: is the sky blue?\"\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + } + ] + }, + { + "name": "31.4 legacy automatic caching — streaming", + "item": [ + { + "name": "pc gpt-4.1-mini nativechat streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4.1-mini\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-gpt-4.1-mini-s-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-gpt-4.1-mini-s]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-4.1-mini nativechat streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4.1-mini\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativechat-gpt-4.1-mini-s-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativechat-gpt-4.1-mini-s]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-4.1-nano nativeresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4.1-nano\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-gpt-4.1-nano-s-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-gpt-4.1-nano-s] Answer in one short word: is the sky blue?\",\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-4.1-nano nativeresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"openai/gpt-4.1-nano\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-nativeresp-gpt-4.1-nano-s-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} nativeresp-gpt-4.1-nano-s] Answer in one short word: is the sky blue?\",\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-4o dropinchat streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4o\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-gpt-4o-s-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-gpt-4o-s]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-4o dropinchat streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4o\",\n \"max_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinchat-gpt-4o-s-{{pcNonce}}\",\n \"messages\": [\n {\n \"role\": \"system\",\n \"content\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinchat-gpt-4o-s]\"\n },\n {\n \"role\": \"user\",\n \"content\": \"Answer in one short word: is the sky blue?\"\n }\n ],\n \"stream\": true,\n \"stream_options\": {\n \"include_usage\": true\n }\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/chat/completions", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "chat", + "completions" + ] + } + } + }, + { + "name": "pc gpt-4o-mini dropinresp streaming [write]", + "event": [ + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream write: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream write: usage present in stream', function(){ pm.expect(r.u,'streamed usage').to.be.an('object'); });", + " pm.test('stream write: cached_tokens present (numeric)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.a('number'); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4o-mini\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-gpt-4o-mini-s-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-gpt-4o-mini-s] Answer in one short word: is the sky blue?\",\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + }, + { + "name": "pc gpt-4o-mini dropinresp streaming [read]", + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [ + "// Give OpenAI's prompt cache time to propagate from the paired [write] before this [read] (option A).", + "var __t = Date.now(); while (Date.now() - __t < 1500) { /* busy-wait ~1.5s for cache propagation */ }" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [ + "var C = pm.response.code;", + "if (C < 400) {", + " var ct = pm.response.headers.get('content-type') || '';", + " pm.test('stream read: SSE content-type', function(){ pm.expect(ct).to.include('text/event-stream'); });", + " function sse(t){ var u=null,p=null; (t||'').split('\\n').forEach(function(l){ l=l.trim(); if(l.indexOf('data:')!==0) return; var s=l.slice(5).trim(); if(!s||s==='[DONE]') return; var o; try{o=JSON.parse(s);}catch(e){return;} if(o.usage)u=o.usage; if(o.response&&o.response.usage)u=o.response.usage; if(o.prompt_cache_options)p=o.prompt_cache_options; if(o.response&&o.response.prompt_cache_options)p=o.response.prompt_cache_options; }); return {u:u,p:p}; }", + " function det(u){ return (u && (u.prompt_tokens_details || u.input_tokens_details)) || {}; } function NUM(v){ return (typeof v === 'number') ? v : 0; } function wtok(d){ return NUM(d.cached_write_tokens) || NUM(d.cache_write_tokens); } function rtok(d){ return NUM(d.cached_tokens); }", + " var r = sse(pm.response.text()); var d = det(r.u);", + " pm.test('stream read: cached_tokens > 0 (cache hit)', function(){ pm.expect(rtok(d),'cached_tokens').to.be.above(0); });", + "}" + ] + } + } + ], + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"model\": \"gpt-4o-mini\",\n \"max_output_tokens\": 8000,\n \"prompt_cache_key\": \"bf-harness-dropinresp-gpt-4o-mini-s-{{pcNonce}}\",\n \"input\": \"{{cachePrefix}} [pc-session {{pcNonce}} dropinresp-gpt-4o-mini-s] Answer in one short word: is the sky blue?\",\n \"stream\": true\n}" + }, + "url": { + "raw": "{{baseUrl}}/openai/v1/responses", + "host": [ + "{{baseUrl}}" + ], + "path": [ + "openai", + "v1", + "responses" + ] + } + } + } + ] + } + ] } ] },