From 78d8e96a15cfd2cdcdd22acca17ae22abb3a482a Mon Sep 17 00:00:00 2001 From: akshaydeo Date: Thu, 30 Jul 2026 02:52:33 +0530 Subject: [PATCH] recalculate cost fixes --- .../logstore/billinghydrationgate_test.go | 404 ++++++++ framework/logstore/billingprojection_test.go | 268 +++++ framework/logstore/contenthidden_test.go | 16 +- framework/logstore/hybrid.go | 232 ++++- framework/logstore/hybrid_test.go | 30 +- framework/logstore/hybridbilling_test.go | 303 ++++++ framework/logstore/migrations.go | 56 ++ framework/logstore/payload.go | 50 + framework/logstore/rdb.go | 180 +++- framework/logstore/store.go | 34 + framework/logstore/tables.go | 71 +- framework/streaming/accumulator.go | 2 + framework/streaming/accumulator_test.go | 30 + framework/streaming/chat.go | 13 + framework/streaming/responses.go | 13 + framework/streaming/responses_test.go | 23 + framework/streaming/types.go | 3 + plugins/logging/costfidelity_test.go | 920 ++++++++++++++++++ plugins/logging/costrecalc.go | 48 +- plugins/logging/costrecalc_test.go | 80 +- plugins/logging/main.go | 8 +- plugins/logging/operations.go | 295 +++++- plugins/logging/operations_test.go | 17 + transports/bifrost-http/handlers/logging.go | 27 +- 24 files changed, 3060 insertions(+), 63 deletions(-) create mode 100644 framework/logstore/billinghydrationgate_test.go create mode 100644 framework/logstore/billingprojection_test.go create mode 100644 framework/logstore/hybridbilling_test.go create mode 100644 plugins/logging/costfidelity_test.go diff --git a/framework/logstore/billinghydrationgate_test.go b/framework/logstore/billinghydrationgate_test.go new file mode 100644 index 00000000000..fc3b9889fb0 --- /dev/null +++ b/framework/logstore/billinghydrationgate_test.go @@ -0,0 +1,404 @@ +package logstore + +import ( + "context" + "path/filepath" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/objectstore" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Object fetches are the expensive part of a recompute: the object store exposes only +// whole-key Get, so recovering ~300 bytes of counters downloads the entire payload, +// message histories and raw bodies included. These tests pin exactly when that happens. +// +// The rule being enforced: fetch if and only if the row is actually missing a pricing +// input that this request type bills on. + +// countingObjectStore records every Get so a test can assert the exact number of +// fetches rather than merely observing the end state. +type countingObjectStore struct { + *objectstore.InMemoryObjectStore + gets []string +} + +func (s *countingObjectStore) Get(ctx context.Context, key string) ([]byte, error) { + s.gets = append(s.gets, key) + return s.InMemoryObjectStore.Get(ctx, key) +} + +func newCountingHybrid(t *testing.T, excludeFields []string) (*HybridLogStore, LogStore, *countingObjectStore) { + t.Helper() + inner, err := newSqliteLogStore( + context.Background(), + &SQLiteConfig{Path: filepath.Join(t.TempDir(), "gate.db")}, + hybridTestLogger{}, + ) + require.NoError(t, err) + objStore := &countingObjectStore{InMemoryObjectStore: objectstore.NewInMemoryObjectStore()} + hybrid := newHybridLogStore(inner, objStore, "test", hybridTestLogger{}, excludeFields) + return hybrid, inner, objStore +} + +// TestBillingRowNeedsHydration is the decision table for the gate. Each case is a row +// shape a real deployment produces. +func TestBillingRowNeedsHydration(t *testing.T) { + const usageJSON = `{"prompt_tokens":10,"completion_tokens":2,"total_tokens":12}` + const cacheJSON = `{"cache_hit":false}` + + excludedSet := func(fields ...string) map[string]struct{} { + set := make(map[string]struct{}, len(fields)) + for _, f := range fields { + set[f] = struct{}{} + } + return set + } + + cases := []struct { + name string + log Log + excluded map[string]struct{} + want bool + why string + }{ + { + name: "everything resident on a chat row", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: cacheJSON}, + want: false, + why: "nothing is missing, so a fetch would recover nothing", + }, + { + name: "token_usage offloaded", + log: Log{Object: "chat.completion", TokenUsage: "", CacheDebug: cacheJSON}, + want: true, + why: "the cache breakdown is the field whose absence caused the overbill", + }, + { + name: "cache_debug empty beside a recovered token_usage", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: ""}, + want: false, + why: "offloading blanks token_usage, so its presence means a prior pass already " + + "fetched and backfilled this row; the empty cache_debug beside it is the real answer", + }, + { + name: "cache_debug offloadable while token_usage is config-resident", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: ""}, + excluded: excludedSet("token_usage"), + want: true, + why: "here token_usage is present only because of config, so it vouches for nothing; " + + "an offloaded semantic-cache hit priced without cache_debug bills $0 as full price", + }, + { + name: "cache_debug empty but configured DB-resident", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: ""}, + excluded: excludedSet("cache_debug"), + want: false, + why: "the column never leaves the DB, so empty is the real answer and the object holds nothing to recover", + }, + { + name: "already hydrated", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: "", billingPayloadsHydrated: true}, + want: false, + why: "the object was already read; re-asking returns the same thing and would refetch forever", + }, + { + name: "speech row missing its own payload", + log: Log{Object: "speech", TokenUsage: usageJSON, CacheDebug: cacheJSON}, + want: true, + why: "TTS bills per input character, which only speech_output carries", + }, + { + name: "speech row with its payload resident", + log: Log{Object: "speech", TokenUsage: usageJSON, CacheDebug: cacheJSON, SpeechOutput: `{"usage":{"input_chars":5}}`}, + want: false, + why: "the per-character input is already present", + }, + { + name: "chat row missing speech_output", + log: Log{Object: "chat.completion", TokenUsage: usageJSON, CacheDebug: cacheJSON}, + want: false, + why: "a chat row does not bill on speech_output, so its absence is not a gap", + }, + { + name: "ocr row missing its own payload", + log: Log{Object: "ocr", TokenUsage: usageJSON, CacheDebug: cacheJSON}, + want: true, + why: "OCR bills per page processed, which only ocr_output carries", + }, + { + name: "image row missing its own payload", + log: Log{Object: "image_generation", TokenUsage: usageJSON, CacheDebug: cacheJSON}, + want: true, + why: "per-image pricing needs the image count from image_generation_output", + }, + { + name: "video row with its payload resident", + log: Log{Object: "video_generation", TokenUsage: usageJSON, CacheDebug: cacheJSON, VideoGenerationOutput: `{"seconds":"8"}`}, + want: false, + why: "the billed duration is already present", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := billingRowNeedsHydration(&tc.log, tc.excluded) + assert.Equal(t, tc.want, got, tc.why) + }) + } +} + +// TestHydrateBillingChunkFetchesOnlyWhenRequired proves the gate end to end by counting +// real Get calls, across the three reasons a fetch is unnecessary. +func TestHydrateBillingChunkFetchesOnlyWhenRequired(t *testing.T) { + t.Run("no fetch when the row was never offloaded", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "inline-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, inner.CreateIfNotExists(ctx, entry)) // straight through: has_object stays false + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + require.Empty(t, hydration.Unpriceable) + assert.Empty(t, objStore.gets, "a row that was never offloaded must not be fetched") + }) + + t.Run("no fetch when the exclusion list kept the billing fields in the DB", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, []string{"token_usage", "cache_debug"}) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "resident-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + CacheDebugParsed: &schemas.BifrostCacheDebug{CacheHit: false}, + InputHistoryParsed: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: new("a long prompt")}}, + }, + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "resident-1") + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + require.True(t, result.Logs[0].HasObject, "the message history should still have been offloaded") + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + require.Empty(t, hydration.Unpriceable) + assert.Empty(t, objStore.gets, + "the row already carries every pricing input; fetching the object would recover nothing") + }) + + // prepareDBEntry keeps token_usage and cache_debug on hidden rows, so a hidden row + // written by the current code is not a reason to download its payload. + t.Run("no fetch for a content-hidden row that kept its pricing metadata", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "hidden-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + ContentHidden: true, TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "hidden-1") + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + require.True(t, result.Logs[0].HasObject, "the hidden payload should still have been offloaded") + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Unpriceable) + assert.Empty(t, objStore.gets, + "hiding content does not remove pricing inputs from the row; fetching the whole payload would recover nothing") + }) + + // Rows written before prepareDBEntry retained pricing metadata: nothing on the row + // can price them, so the retained object is the only source. + t.Run("one billing-only fetch for a legacy content-hidden row", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "hidden-legacy-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + ContentHidden: true, TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "hidden-legacy-1") + // Reproduce the old write path, which blanked pricing metadata along with the + // rest of the payload. + require.NoError(t, inner.Update(ctx, "hidden-legacy-1", map[string]interface{}{ + "token_usage": "", + })) + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Unpriceable) + assert.Equal(t, []string{"hidden-legacy-1"}, hydration.Hydrated) + assert.Len(t, objStore.gets, 1, + "billing may recover retained pricing inputs without changing serving-path visibility") + }) + + t.Run("exactly one fetch when token_usage was offloaded", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "offloaded-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "offloaded-1") + require.NoError(t, inner.Update(ctx, "offloaded-1", map[string]interface{}{ + "token_usage": "", + "cache_debug": "", + })) + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + require.Empty(t, result.Logs[0].TokenUsage, "precondition: token_usage was offloaded") + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + require.Empty(t, hydration.Unpriceable) + assert.Equal(t, []string{"offloaded-1"}, hydration.Hydrated, "and it must report what it fetched, so the caller can back it up") + assert.Len(t, objStore.gets, 1, "one fetch, and only one, for a genuinely degraded row") + assert.False(t, result.Logs[0].IsUsageDegraded(), "and it must have actually recovered the payload") + }) + + t.Run("a second hydration of the same row does not refetch", func(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "idem-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "idem-1") + require.NoError(t, inner.Update(ctx, "idem-1", map[string]interface{}{ + "token_usage": "", + "cache_debug": "", + })) + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + row := &result.Logs[0] + + _, err = hybrid.HydrateBillingChunk(ctx, []*Log{row}) + require.NoError(t, err) + _, err = hybrid.HydrateBillingChunk(ctx, []*Log{row}) + require.NoError(t, err) + assert.Len(t, objStore.gets, 1, + "once hydrated the row carries its inputs, so the gate must skip it on a repeat call") + }) +} + +// TestBackfillMakesLaterHydrationFetchFree is the convergence property, proven against +// a real hybrid store: after the recovered pricing inputs are written back, a fresh read +// of the same row needs no object fetch at all. +// +// This is what makes recompute self-healing — the first pass pays the fetches, every +// pass after it is served from the database. +func TestBackfillMakesLaterHydrationFetchFree(t *testing.T) { + hybrid, inner, objStore := newCountingHybrid(t, nil) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "converge-1", Timestamp: time.Now().UTC(), Provider: "anthropic", + Model: "claude-sonnet-4-20250514", Status: "success", Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + InputHistoryParsed: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: new("a long prompt")}}, + }, + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "converge-1") + require.NoError(t, inner.Update(ctx, "converge-1", map[string]interface{}{ + "token_usage": "", + "cache_debug": "", + })) + + // First pass: the row arrives degraded and one fetch recovers it. + first, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, first.Logs, 1) + require.Empty(t, first.Logs[0].TokenUsage, "precondition: token_usage was offloaded") + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&first.Logs[0]}) + require.NoError(t, err) + require.Equal(t, []string{"converge-1"}, hydration.Hydrated) + require.Len(t, objStore.gets, 1) + + // Persist what was recovered, the way the recalc job does. + require.NoError(t, hybrid.BulkBackfillBillingPayloads(ctx, map[string]BillingPayloadBackfill{ + "converge-1": {TokenUsage: first.Logs[0].TokenUsage, CacheDebug: first.Logs[0].CacheDebug}, + })) + + // Second pass: a completely fresh read of the same row. + second, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, second.Logs, 1) + require.NotEmpty(t, second.Logs[0].TokenUsage, "the backfill should have landed in the row") + assert.False(t, second.Logs[0].IsUsageDegraded(), "and it should parse as a real payload, not a stub") + + hydration, err = hybrid.HydrateBillingChunk(ctx, []*Log{&second.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Hydrated) + assert.Empty(t, hydration.Unpriceable) + assert.Len(t, objStore.gets, 1, + "the second pass must be served from the database; the fetch count must not grow") + + // The full breakdown survived the round trip, which is what pricing depends on. + details := second.Logs[0].TokenUsageParsed.PromptTokensDetails + require.NotNil(t, details) + assert.Equal(t, 70_000, details.CachedReadTokens) + assert.Equal(t, 10_000, details.CachedWriteTokens) + require.NotNil(t, details.CachedWriteTokenDetails) + assert.Equal(t, 4_000, details.CachedWriteTokenDetails.CachedWriteTokens1h) + + // Offloading is not undone: the row still points at its object, and the DB keeps only + // the last-user-message preview prepareDBEntry leaves behind, not the full history. + dbRow, err := inner.FindByID(ctx, "converge-1") + require.NoError(t, err) + assert.True(t, dbRow.HasObject, "the row must still reference its offloaded payload") + assert.Equal(t, 1, objStore.Len(), "the object must still hold the full payload") +} diff --git a/framework/logstore/billingprojection_test.go b/framework/logstore/billingprojection_test.go new file mode 100644 index 00000000000..3b701e9db48 --- /dev/null +++ b/framework/logstore/billingprojection_test.go @@ -0,0 +1,268 @@ +package logstore + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The list projection and the billing projection answer different questions and must +// not converge. +// +// listSelectColumns serves /api/logs. Its comment records a real incident: including +// unbounded TEXT payloads pushed the response past Cloud Run's 32MB body limit and +// returned 500s. So it must stay free of the output blobs. +// +// billingSelectColumns serves cost recomputation, which reads those blobs to recover +// audio duration, TTS character counts, image counts and OCR pages — but it must carry +// nothing else, because a recompute holds a whole batch in memory at once. + +// newProjectionTestStore builds a real sqlite-backed store so the projection tests +// can actually issue the queries. A temp file rather than ":memory:" because sqlite +// gives each connection its own in-memory database, and the store's pool uses more +// than one — the same reason newTestHybrid takes this approach. +func newProjectionTestStore(t *testing.T) *RDBLogStore { + t.Helper() + store, err := newSqliteLogStore( + context.Background(), + &SQLiteConfig{Path: filepath.Join(t.TempDir(), "projection.db")}, + testLogger{}, + ) + require.NoError(t, err) + return store +} + +func TestListProjectionExcludesOutputPayloadBlobs(t *testing.T) { + cols := newProjectionTestStore(t).listSelectColumns() + for blob := range billingPayloadColumns { + if containsColumn(cols, blob) { + t.Fatalf("listSelectColumns must not select %q: it serves /api/logs, where unbounded "+ + "payload columns reintroduce the Cloud Run 32MB body-limit failure", blob) + } + } +} + +// TestListProjectionIncludesBillingScalars pins the columns whose absence produced the +// production overbill. cached_read_tokens gates the degraded-usage rebuild in +// DeserializeFields, and has_object / content_hidden decide whether a row can be +// hydrated at all. They are cheap scalars, so the list path carries them too. +func TestListProjectionIncludesBillingScalars(t *testing.T) { + cols := newProjectionTestStore(t).listSelectColumns() + for _, scalar := range []string{ + "cached_read_tokens", + "has_object", + "content_hidden", + "service_tier", + "speed", + "inference_geo", + } { + if !containsColumn(cols, scalar) { + t.Fatalf("listSelectColumns is missing %q; cost recomputation cannot see it", scalar) + } + } +} + +func TestBillingProjectionSelectsEveryPricingInput(t *testing.T) { + cols := newProjectionTestStore(t).billingSelectColumns() + for _, scalar := range billingScalarColumns { + if !containsColumn(cols, scalar) { + t.Fatalf("billingSelectColumns is missing %q; pricing reads it", scalar) + } + } +} + +// TestBillingProjectionGatesPayloadsByObjectType is the memory guard. +// +// image_generation_output serializes ImageData.B64JSON — full base64 image bytes, +// megabytes per image — while pricing only ever reads len(Data). Selecting these +// unconditionally would pull that for every row in a batch, including plain chat rows. +func TestBillingProjectionGatesPayloadsByObjectType(t *testing.T) { + cols := newProjectionTestStore(t).billingSelectColumns() + + for blob, objectTypes := range billingPayloadColumns { + if !strings.Contains(cols, "AS "+blob) { + t.Fatalf("billingSelectColumns is missing %q; the modality patch-ins in "+ + "calculateCostForLog cannot restore per-second / per-page / per-image billing without it", blob) + } + if containsColumn(cols, blob) { + t.Fatalf("%q is selected unconditionally; it must be gated on object_type so a "+ + "chat row never pulls an unrelated payload into memory", blob) + } + for _, objectType := range objectTypes { + if !strings.Contains(cols, "'"+objectType+"'") { + t.Fatalf("%q is not gated for object_type %q, so those rows would lose their billing payload", blob, objectType) + } + } + } +} + +// TestBillingProjectionOmitsColumnsPricingNeverReads enforces "only what is absolutely +// required". A recompute batch is held entirely in memory, so anything pricing does not +// read is pure ballast — and these are the expensive ones. +func TestBillingProjectionOmitsColumnsPricingNeverReads(t *testing.T) { + cols := newProjectionTestStore(t).billingSelectColumns() + for _, unused := range []string{ + "input_history", + "responses_input_history", + "output_message", + "content_summary", + "metadata", + "speech_input", + "transcription_input", + "image_generation_input", + "video_generation_input", + } { + if strings.Contains(cols, unused) { + t.Fatalf("billingSelectColumns pulls %q, which pricing never reads; a recompute "+ + "holds the whole batch at once so this is wasted memory", unused) + } + } +} + +// TestBillingProjectionDeliversModalityOutputsEndToEnd is the query-level counterpart +// to the string assertions above. +// +// calculateCostForLog patches audio duration, TTS character counts, image counts, and +// OCR pages back onto the reconstructed response by reading these Parsed fields. Since +// listSelectColumns never selected the underlying columns, those patch-ins were dead +// code on the recompute path and every one of those request types was priced off +// aggregate token counts alone. +func TestBillingProjectionDeliversModalityOutputsEndToEnd(t *testing.T) { + store := newProjectionTestStore(t) + ctx := context.Background() + + annotation := "annotated" + seconds := "8" + // object_type drives the CASE gate, so each payload needs a row of its own type. + rows := []*Log{ + { + ID: "speech-1", Object: "speech", Provider: "openai", Model: "gpt-4o-mini-tts", + SpeechOutputParsed: &schemas.BifrostSpeechResponse{ + Usage: &schemas.SpeechUsage{InputTokens: 40, InputChars: 512, TotalTokens: 40}, + }, + }, + { + ID: "transcription-1", Object: "transcription", Provider: "openai", Model: "whisper-1", + TranscriptionOutputParsed: &schemas.BifrostTranscriptionResponse{ + Usage: &schemas.TranscriptionUsage{Seconds: new(float64(31))}, + }, + }, + { + ID: "video-1", Object: "video_generation", Provider: "openai", Model: "sora", + VideoGenerationOutputParsed: &schemas.BifrostVideoGenerationResponse{Seconds: &seconds}, + }, + { + ID: "ocr-1", Object: "ocr", Provider: "mistral", Model: "mistral-ocr", + OCROutputParsed: &schemas.BifrostOCRResponse{ + Pages: []schemas.OCRPage{{}, {}, {}}, + UsageInfo: &schemas.OCRUsageInfo{PagesProcessed: 3}, + DocumentAnnotation: &annotation, + }, + }, + } + for i, r := range rows { + r.Timestamp = time.Now().UTC().Add(time.Duration(i) * time.Second) + r.Status = "success" + require.NoError(t, r.SerializeFields()) + require.NoError(t, store.Create(ctx, r)) + } + + listResult, err := store.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, listResult.Logs, len(rows)) + for _, got := range listResult.Logs { + assert.Nil(t, got.OCROutputParsed, + "the list projection must keep omitting the output blobs; /api/logs has a body-size limit to respect") + assert.Nil(t, got.SpeechOutputParsed) + } + + billingResult, err := store.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, billingResult.Logs, len(rows)) + + byID := make(map[string]*Log, len(billingResult.Logs)) + for i := range billingResult.Logs { + byID[billingResult.Logs[i].ID] = &billingResult.Logs[i] + } + + speech := byID["speech-1"] + require.NotNil(t, speech.SpeechOutputParsed, "TTS per-character billing needs speech_output") + assert.Equal(t, 512, speech.SpeechOutputParsed.Usage.InputChars) + assert.Nil(t, speech.OCROutputParsed, "a speech row must not pull the OCR payload") + + transcription := byID["transcription-1"] + require.NotNil(t, transcription.TranscriptionOutputParsed, "per-second audio billing needs transcription_output") + require.NotNil(t, transcription.TranscriptionOutputParsed.Usage.Seconds) + assert.Equal(t, float64(31), *transcription.TranscriptionOutputParsed.Usage.Seconds) + + video := byID["video-1"] + require.NotNil(t, video.VideoGenerationOutputParsed, "per-second video billing needs video_generation_output") + assert.Equal(t, "8", *video.VideoGenerationOutputParsed.Seconds) + + ocr := byID["ocr-1"] + require.NotNil(t, ocr.OCROutputParsed, "per-page OCR billing needs ocr_output") + assert.Equal(t, 3, ocr.OCROutputParsed.UsageInfo.PagesProcessed) + require.NotNil(t, ocr.OCROutputParsed.DocumentAnnotation, "the annotation surcharge is keyed off this field") +} + +// TestBillingProjectionStripsGeneratedImageBytes pins the OOM guard. Pricing needs the +// image count, never the images: extractCostInput passes len(Data) to +// populateOutputImageCount and nothing reads B64JSON, which can be megabytes per image. +func TestBillingProjectionStripsGeneratedImageBytes(t *testing.T) { + store := newProjectionTestStore(t) + ctx := context.Background() + + size := "1024x1024" + entry := &Log{ + ID: "image-1", + Timestamp: time.Now().UTC(), + Object: "image_generation", + Provider: "openai", + Model: "gpt-image-1", + Status: "success", + ImageGenerationOutputParsed: &schemas.BifrostImageGenerationResponse{ + Data: []schemas.ImageData{ + {B64JSON: strings.Repeat("A", 4096), RevisedPrompt: "a cat"}, + {B64JSON: strings.Repeat("B", 4096)}, + }, + ImageGenerationResponseParameters: &schemas.ImageGenerationResponseParameters{Size: size}, + }, + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, store.Create(ctx, entry)) + + result, err := store.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + got := result.Logs[0] + + require.NotNil(t, got.ImageGenerationOutputParsed) + assert.Len(t, got.ImageGenerationOutputParsed.Data, 2, + "the image count drives per-image pricing and must survive") + require.NotNil(t, got.ImageGenerationOutputParsed.ImageGenerationResponseParameters) + assert.Equal(t, size, got.ImageGenerationOutputParsed.Size, "size drives per-pixel pricing") + + for i, img := range got.ImageGenerationOutputParsed.Data { + assert.Empty(t, img.B64JSON, "generated image bytes must be released at index %d; pricing never reads them", i) + } + assert.Empty(t, got.ImageGenerationOutput, + "the serialized column still holds the base64 and must be released once parsed") +} + +// containsColumn matches a bare column name in a SELECT clause. It deliberately does +// not match a name that only appears inside a CASE expression, so the gating tests can +// tell "selected unconditionally" from "selected for its own object_type". +func containsColumn(clause, column string) bool { + for _, part := range strings.Split(clause, ",") { + if strings.TrimSpace(part) == column { + return true + } + } + return false +} diff --git a/framework/logstore/contenthidden_test.go b/framework/logstore/contenthidden_test.go index 77fa36f5066..eb237cca18b 100644 --- a/framework/logstore/contenthidden_test.go +++ b/framework/logstore/contenthidden_test.go @@ -38,13 +38,19 @@ func TestHybrid_ContentHiddenStripsDBRowAndSkipsHydration(t *testing.T) { entry := newContentHiddenTestEntry("hidden-1") entry.ContentHidden = true + entry.TokenUsageParsed = billingTestUsage() require.NoError(t, entry.SerializeFields()) require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) - waitForUploads(t, func() bool { return objStore.Len() == 1 }) - - // The DB row must hold no content at all: no payload fields, no summary, - // no last-user-message preview. + // Wait for the has_object flag, not just the object: processUpload writes the + // flag after the Put, so waiting on objStore.Len() alone races the assertion + // below under parallel load. + waitForOffload(t, inner, "hidden-1") + require.Equal(t, 1, objStore.Len()) + + // The DB row must hold no request/response content: no content payload fields, + // summary, or last-user-message preview. Pricing metadata is not content and + // remains DB-resident. dbRow, err := inner.FindByID(ctx, "hidden-1") require.NoError(t, err) assert.True(t, dbRow.ContentHidden) @@ -52,6 +58,8 @@ func TestHybrid_ContentHiddenStripsDBRowAndSkipsHydration(t *testing.T) { assert.Empty(t, dbRow.InputHistory) assert.Empty(t, dbRow.OutputMessage) assert.Empty(t, dbRow.ContentSummary) + assert.NotEmpty(t, dbRow.TokenUsage) + require.NotNil(t, dbRow.TokenUsageParsed) // The full payload must be in object storage. data, err := objStore.Get(ctx, ObjectKey("test", entry.Timestamp, "hidden-1")) diff --git a/framework/logstore/hybrid.go b/framework/logstore/hybrid.go index 6a24700eb44..04b811bbb84 100644 --- a/framework/logstore/hybrid.go +++ b/framework/logstore/hybrid.go @@ -231,14 +231,27 @@ func (h *HybridLogStore) enqueueRawUpload(logID string, timestamp time.Time, key // prepareDBEntry builds the lightweight DB entry by extracting the content // summary, trimming input history to the last user message, and clearing // payload fields that will be offloaded to object storage. Fields in the -// excluded set are kept intact in the DB row. +// excluded set are kept intact in the DB row. Pricing metadata is always kept +// DB-resident regardless of hybrid configuration or content visibility. // Must be called after SerializeFields() populates the Parsed fields. func prepareDBEntry(dbEntry *Log, excluded map[string]struct{}) { - // Hidden entries keep no content in the DB row at all: no summary, no - // last-user-message preview, and no exclusion-list carve-outs. The full - // payload lives only in object storage and is never hydrated back. + tokenUsage := dbEntry.TokenUsage + cacheDebug := dbEntry.CacheDebug + tokenUsageParsed := dbEntry.TokenUsageParsed + cacheDebugParsed := dbEntry.CacheDebugParsed + restorePricingMetadata := func() { + dbEntry.TokenUsage = tokenUsage + dbEntry.CacheDebug = cacheDebug + dbEntry.TokenUsageParsed = tokenUsageParsed + dbEntry.CacheDebugParsed = cacheDebugParsed + } + + // Hidden entries keep no request/response content in the DB row: no summary, + // last-user-message preview, or exclusion-list carve-outs. Pricing metadata is + // not content and remains in the log store. if dbEntry.ContentHidden { ClearPayload(dbEntry) + restorePricingMetadata() dbEntry.ContentSummary = "" return } @@ -279,6 +292,7 @@ func prepareDBEntry(dbEntry *Log, excluded map[string]struct{}) { } ClearPayloadFiltered(dbEntry, excluded) + restorePricingMetadata() if _, hasInputHistoryExclusion := excluded["input_history"]; !hasInputHistoryExclusion { dbEntry.InputHistory = sanitizeJSONForJSONB(lastUserMessage) @@ -636,6 +650,216 @@ func (h *HybridLogStore) SearchLogs(ctx context.Context, filters SearchFilters, return h.inner.SearchLogs(ctx, filters, pagination) } +// SearchLogsForBilling delegates to the inner store. Hydration is deliberately NOT +// done here — see HydrateBillingChunk. +func (h *HybridLogStore) SearchLogsForBilling(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) { + return h.inner.SearchLogsForBilling(ctx, filters, pagination) +} + +// HydrateBillingChunk restores offloaded pricing inputs for a small slice of rows. +// +// This exists because of a real mis-billing bug: a row whose token_usage was offloaded +// comes back with a stub rebuilt from denormalized columns, and because PromptTokens is +// inclusive of the cache buckets, pricing that stub charges every cache-read token at +// the full input rate — inflating a cache-heavy request several fold. +// +// Two deliberate properties: +// +// SEQUENTIAL — one object in flight at a time. A parallel fan-out multiplies peak +// memory by its width, since each in-flight fetch holds the whole object as a []byte +// plus the map[string]string that MergePayloadFromJSON unmarshals it into, and objects +// can carry full message histories and raw request/response bodies. This runs inside a +// checkpointed background job, so predictability beats throughput. Combined with the +// caller paging and chunking at BillingHydrationChunkSize, a recompute worker's +// payload memory is bounded by a handful of rows no matter how large the window. +// +// SKIPS ROWS THAT DO NOT NEED IT — a row already carrying its pricing inputs costs no +// I/O at all. See billingRowNeedsHydration. +func (h *HybridLogStore) HydrateBillingChunk(ctx context.Context, logs []*Log) (BillingHydrationResult, error) { + var result BillingHydrationResult + for _, log := range logs { + if log == nil || !log.HasObject { + // Never offloaded: the row already carries its own payload. + continue + } + if !billingRowNeedsHydration(log, h.excludedPayloadFields) { + continue + } + // Checked per row rather than relying on Get to notice: on shutdown or job + // cancellation this should stop fetching immediately, not walk the rest of + // the chunk collecting context errors. + if err := ctx.Err(); err != nil { + return result, err + } + if hydrateErr := h.hydrateLogForBilling(ctx, log); hydrateErr != nil { + h.logger.Warn("objectstore: cannot hydrate pricing inputs for log %s: %v", log.ID, hydrateErr) + result.Unpriceable = append(result.Unpriceable, log.ID) + continue + } + result.Hydrated = append(result.Hydrated, log.ID) + } + return result, nil +} + +// billingRowNeedsHydration reports whether an offloaded row is actually missing +// something pricing reads. Fetching a whole object to recover fields the row already +// has is pure waste — and because the store exposes only whole-key Get, one wasted +// fetch downloads the entire payload, message histories included. +// +// excluded is the store's object_storage_exclude_fields set, and it is what makes the +// check precise. An empty column on its own is ambiguous: it can mean "offloaded" or +// "never written". A row with no semantic cache has an empty cache_debug forever, so +// treating empty as "go fetch" would fetch nearly every row and would keep fetching +// the same row after a successful hydration recovered nothing. Consulting the +// exclusion set resolves that: if a column is configured DB-resident, empty means +// genuinely empty and there is nothing in the object to recover. +// +// The fields considered: +// +// - token_usage: the field whose absence caused the overbill. +// - cache_debug: absent means a semantic-cache hit could be priced at full LLM cost +// instead of $0, so it is fetched unless configured resident. +// - the modality payload: only the column this row's own request type bills on. A +// chat row is not missing anything by having no speech_output. +func billingRowNeedsHydration(l *Log, excluded map[string]struct{}) bool { + // Already fetched in this pass: whatever the object held is now on the row, and the + // fields it did not hold will never appear no matter how often we ask. + if l.billingPayloadsHydrated { + return false + } + // Content-hidden rows are eligible for billing-only hydration even though ordinary + // serving reads never hydrate them — but eligible is not the same as force-fetched. + // They get their own branch rather than falling through because they are written with + // ClearPayload, which ignores the DB-resident exclusion list entirely, so `excluded` + // vouches for nothing on them and the checks below would misread it in both + // directions: it would wave through an emptied-but-"resident" modality column, and it + // would defeat the backfilled inference for a deployment that keeps token_usage + // resident, refetching such a row on every pass to recover nothing. + // + // A stronger signal replaces it. prepareDBEntry restores token_usage and cache_debug + // together on every hidden row, so a present token_usage means this row was written + // after that change and its empty cache_debug is genuinely empty. A blank one is a + // legacy hidden row whose pricing inputs live only in the retained object. + if l.ContentHidden { + if l.TokenUsage == "" { + return true + } + // Modality payloads are cleared unconditionally here and are never backfilled, + // so an empty column means the object holds the only copy. + if col := billingPayloadColumnFor(l.Object); col != "" { + return billingPayloadColumnValue(l, col) == "" + } + return false + } + + resident := func(column string) bool { + _, ok := excluded[column] + return ok + } + + // A previous recompute already fetched this row and wrote the small pricing inputs + // back (see BulkBackfillBillingPayloads). That state is detectable without a marker + // column: offloading blanks token_usage, so a has_object row that nonetheless has + // one can only have been backfilled. The exception is a deployment that configured + // token_usage to stay resident — there its presence is just config and says nothing + // about whether anything else was recovered. + // + // This matters because writing an empty cache_debug back cannot speak for itself: + // most rows have no semantic cache, so without this inference the gate would keep + // refetching them forever and the backfill would buy nothing. + backfilled := l.TokenUsage != "" && !resident("token_usage") + + vouched := func(column, value string) bool { + return value != "" || resident(column) || backfilled + } + + if !vouched("token_usage", l.TokenUsage) || !vouched("cache_debug", l.CacheDebug) { + return true + } + if col := billingPayloadColumnFor(l.Object); col != "" { + // Modality payloads are never backfilled — they are precisely what offloading + // exists to keep out of the database — so only the exclusion set can vouch for + // an empty column here, never a prior backfill. + return billingPayloadColumnValue(l, col) == "" && !resident(col) + } + return false +} + +// BulkBackfillBillingPayloads delegates to the inner store. +// +// Safe to write these columns even though the offload path blanked them: Update is a +// plain pass-through here (it does not re-run prepareDBEntry), so nothing re-strips +// them. The object keeps its own copy, so this is a cache, not a move. +func (h *HybridLogStore) BulkBackfillBillingPayloads(ctx context.Context, updates map[string]BillingPayloadBackfill) error { + return h.inner.BulkBackfillBillingPayloads(ctx, updates) +} + +// billingPayloadColumnValue reads the serialized value of one modality payload column. +func billingPayloadColumnValue(l *Log, column string) string { + switch column { + case "speech_output": + return l.SpeechOutput + case "transcription_output": + return l.TranscriptionOutput + case "image_generation_output": + return l.ImageGenerationOutput + case "video_generation_output": + return l.VideoGenerationOutput + case "ocr_output": + return l.OCROutput + default: + return "" + } +} + +// hydrateLogForBilling fetches the offloaded payload and merges back only the +// fields pricing reads, then re-deserializes so the virtual structs match. +// +// Unlike hydrateLog it returns the error instead of degrading gracefully. Silent +// degradation is right for a rendering path — a missing message preview is +// cosmetic — but here it would leave the lossy usage stub in place and bill it, +// which is exactly the failure this method exists to prevent. +func (h *HybridLogStore) hydrateLogForBilling(ctx context.Context, log *Log) error { + // Only the payload this row's own request type bills on is kept; the rest are + // pruned after merge so an unrelated blob never lingers in memory. + fields := []string{"token_usage", "cache_debug"} + if col := billingPayloadColumnFor(log.Object); col != "" { + fields = append(fields, col) + } + + data, err := h.objects.Get(ctx, ObjectKey(h.prefix, log.Timestamp, log.ID)) + if err != nil { + return fmt.Errorf("fetch payload: %w", err) + } + if err := MergePayloadFromJSON(log, data); err != nil { + return fmt.Errorf("merge payload: %w", err) + } + pruneUnrequestedPayloadFields(log, fields) + // MergePayloadFromJSON writes the serialized columns only. Re-deserialize so + // TokenUsageParsed reflects the hydrated payload rather than the stub AfterFind + // built while token_usage was still blank. + if err := log.DeserializeFields(); err != nil { + return fmt.Errorf("deserialize hydrated payload: %w", err) + } + // Recorded before the usage check below: the object was read, so re-asking would + // return the same thing. Only a fetch that never happened should be retried. + log.billingPayloadsHydrated = true + // Drop the generated-image bytes the hydrated payload carries but pricing never + // reads; it only needs the image count. + stripNonBillingPayloadBytes(log) + // Only token-billed rows need token_usage. A speech/ocr/video/image row prices off + // the modality payload recovered above — input chars, pages processed, seconds, + // image count — and computeOCRCost takes no usage argument at all. Rejecting those + // here would report Unpriceable, which claims the inputs are unrecoverable, when + // hydration in fact recovered everything pricing reads. + if log.TokenUsage == "" && billingPayloadColumnFor(log.Object) == "" { + // The object exists but carries no usage. Nothing to price from, and the + // stub is still in place — report rather than let the caller bill it. + return fmt.Errorf("hydrated payload has no token_usage") + } + return nil +} + // GetSessionLogs delegates to the inner store and returns the paginated logs // belonging to the given session. func (h *HybridLogStore) GetSessionLogs(ctx context.Context, sessionID string, pagination PaginationOptions) (*SessionDetailResult, error) { diff --git a/framework/logstore/hybrid_test.go b/framework/logstore/hybrid_test.go index e065483a67a..5269c72102a 100644 --- a/framework/logstore/hybrid_test.go +++ b/framework/logstore/hybrid_test.go @@ -44,7 +44,7 @@ func newTestHybrid(t *testing.T) (*HybridLogStore, LogStore, *objectstore.InMemo func waitForUploads(t *testing.T, done func() bool) { t.Helper() - deadline := time.Now().Add(2 * time.Second) + deadline := time.Now().Add(5 * time.Second) for time.Now().Before(deadline) { if done() { return @@ -54,6 +54,21 @@ func waitForUploads(t *testing.T, done func() bool) { t.Fatal("timed out waiting for upload state") } +// waitForOffload waits until the offload is fully complete for id: the payload is in +// object storage AND the row's has_object flag has been committed. +// +// processUpload does the Put first and only then updates has_object (with retries), so +// waiting on the object store alone returns while the DB flag is still false. Any test +// that reads has_object — or exercises a code path that branches on it, such as billing +// hydration — must wait for the flag, not just the object. +func waitForOffload(t *testing.T, inner LogStore, id string) { + t.Helper() + waitForUploads(t, func() bool { + row, err := inner.FindByID(context.Background(), id) + return err == nil && row.HasObject + }) +} + func TestHybridScopedDBDelegatesToInnerRDBStore(t *testing.T) { hybrid, _, _ := newTestHybrid(t) defer hybrid.Close(context.Background()) @@ -858,10 +873,9 @@ func TestHybrid_AttachmentsStrippedFromResponsesPreview(t *testing.T) { } func TestHybrid_TokenUsageSummaryForListPreview(t *testing.T) { - // token_usage is offloaded to object storage and cleared from the DB row. The - // denormalized prompt/completion/total columns must remain so list queries can - // rebuild token_usage for the UI without hydrating from S3 (same pattern as - // content_summary for message previews). + // Pricing metadata stays in the DB even when content is offloaded. List queries + // may still use their lightweight projection and rebuild a usage summary from + // denormalized counters. hybrid, inner, objStore := newTestHybrid(t) defer hybrid.Close(context.Background()) ctx := context.Background() @@ -878,6 +892,7 @@ func TestHybrid_TokenUsageSummaryForListPreview(t *testing.T) { CompletionTokens: 45, TotalTokens: 165, }, + CacheDebugParsed: &schemas.BifrostCacheDebug{CacheHit: true}, } require.NoError(t, entry.SerializeFields()) require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) @@ -885,7 +900,8 @@ func TestHybrid_TokenUsageSummaryForListPreview(t *testing.T) { dbLog, err := inner.FindByID(ctx, "chat-tokens-1") require.NoError(t, err) - assert.Empty(t, dbLog.TokenUsage, "token_usage should be offloaded to object storage") + assert.NotEmpty(t, dbLog.TokenUsage, "pricing metadata must stay in log_store") + assert.NotEmpty(t, dbLog.CacheDebug, "cache pricing metadata must stay in log_store") assert.Equal(t, 120, dbLog.PromptTokens) assert.Equal(t, 45, dbLog.CompletionTokens) assert.Equal(t, 165, dbLog.TotalTokens) @@ -901,7 +917,7 @@ func TestHybrid_TokenUsageSummaryForListPreview(t *testing.T) { found, err := hybrid.FindByID(ctx, "chat-tokens-1") require.NoError(t, err) - require.NotNil(t, found.TokenUsageParsed, "detail view should hydrate full token_usage from S3") + require.NotNil(t, found.TokenUsageParsed, "detail view should retain full token_usage") assert.Equal(t, 165, found.TokenUsageParsed.TotalTokens) } diff --git a/framework/logstore/hybridbilling_test.go b/framework/logstore/hybridbilling_test.go new file mode 100644 index 00000000000..00f4125caaf --- /dev/null +++ b/framework/logstore/hybridbilling_test.go @@ -0,0 +1,303 @@ +package logstore + +import ( + "context" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// SearchLogs and SearchLogsForBilling read the same rows at deliberately different +// fidelities, and conflating them caused a production overbill. +// +// SearchLogs returns a usage stub rebuilt from denormalized columns — fine for +// rendering token counts in the list, useless for money. PromptTokens is inclusive of +// the cache buckets, so pricing the stub charges every cache-read token at the full +// input rate; on a 70%-cache-read request that is roughly 2.5x, and a user saw ~3x. +// +// SearchLogsForBilling must therefore hydrate the real payload, and must report +// anything it cannot hydrate instead of quietly handing back the stub. + +func billingTestUsage() *schemas.BifrostLLMUsage { + return &schemas.BifrostLLMUsage{ + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 70_000, + CachedWriteTokens: 10_000, + CachedWriteTokenDetails: &schemas.ChatCachedWriteTokenDetails{ + CachedWriteTokens1h: 4_000, + }, + }, + CompletionTokensDetails: &schemas.ChatCompletionTokensDetails{ + ReasoningTokens: 120, + }, + } +} + +func seedOffloadedBillingLog(t *testing.T, hybrid *HybridLogStore, inner LogStore, id string, contentHidden bool) *Log { + t.Helper() + entry := &Log{ + ID: id, + Timestamp: time.Now().UTC(), + Provider: "anthropic", + Model: "claude-sonnet-4-20250514", + Status: "success", + Object: "chat.completion", + ContentHidden: contentHidden, + TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(context.Background(), entry)) + // Wait for has_object, not just the object: SearchLogsForBilling branches on + // that flag to decide whether a row needs hydrating, and processUpload commits + // it only after the Put. + waitForOffload(t, inner, id) + // Simulate a legacy row written before pricing metadata became unconditionally + // DB-resident. The retained object still has the original payload. + require.NoError(t, inner.Update(context.Background(), id, map[string]interface{}{ + "token_usage": "", + "cache_debug": "", + })) + return entry +} + +func TestHybrid_SearchLogsForBillingHydratesCacheBreakdown(t *testing.T) { + hybrid, inner, objStore := newTestHybrid(t) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + seedOffloadedBillingLog(t, hybrid, inner, "bill-hydrate-1", false) + require.Equal(t, 1, objStore.Len()) + + // Precondition: the DB row really did lose its payload, so the hydration below is + // doing the work rather than the column just happening to still be there. + dbLog, err := inner.FindByID(ctx, "bill-hydrate-1") + require.NoError(t, err) + require.Empty(t, dbLog.TokenUsage, "token_usage should be offloaded to object storage") + + // The list path yields the lossy stub and flags itself as such. + listResult, err := hybrid.SearchLogs(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, listResult.Logs, 1) + stub := listResult.Logs[0] + require.NotNil(t, stub.TokenUsageParsed) + assert.True(t, stub.IsUsageDegraded(), "the list rebuild must mark itself unfit for billing") + assert.Nil(t, stub.TokenUsageParsed.CompletionTokensDetails, + "the stub cannot carry completion details; only billing hydration recovers them") + + // The billing path recovers every pricing input. + billingResult, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, billingResult.Logs, 1) + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&billingResult.Logs[0]}) + require.NoError(t, err) + require.Empty(t, hydration.Unpriceable) + require.Equal(t, []string{"bill-hydrate-1"}, hydration.Hydrated, + "the store must report what it fetched so the caller can back it up") + + hydrated := billingResult.Logs[0] + assert.False(t, hydrated.IsUsageDegraded(), "a hydrated row must be priceable") + require.NotNil(t, hydrated.TokenUsageParsed) + require.NotNil(t, hydrated.TokenUsageParsed.PromptTokensDetails) + assert.Equal(t, 70_000, hydrated.TokenUsageParsed.PromptTokensDetails.CachedReadTokens) + assert.Equal(t, 10_000, hydrated.TokenUsageParsed.PromptTokensDetails.CachedWriteTokens) + require.NotNil(t, hydrated.TokenUsageParsed.PromptTokensDetails.CachedWriteTokenDetails, + "the 5m/1h split drives the above-1hr cache-write rate") + assert.Equal(t, 4_000, hydrated.TokenUsageParsed.PromptTokensDetails.CachedWriteTokenDetails.CachedWriteTokens1h) + require.NotNil(t, hydrated.TokenUsageParsed.CompletionTokensDetails) + assert.Equal(t, 120, hydrated.TokenUsageParsed.CompletionTokensDetails.ReasoningTokens) +} + +// TestHybrid_SearchLogsForBillingHydratesContentHiddenForBilling keeps serving reads +// metadata-only while allowing the internal billing path to recover retained pricing +// inputs from object storage. +func TestHybrid_SearchLogsForBillingHydratesContentHiddenForBilling(t *testing.T) { + hybrid, inner, objStore := newTestHybrid(t) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + seedOffloadedBillingLog(t, hybrid, inner, "bill-hidden-1", true) + require.Equal(t, 1, objStore.Len()) + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Unpriceable) + assert.Equal(t, []string{"bill-hidden-1"}, hydration.Hydrated) + assert.False(t, result.Logs[0].IsUsageDegraded()) + require.NotNil(t, result.Logs[0].TokenUsageParsed) + require.NotNil(t, result.Logs[0].TokenUsageParsed.PromptTokensDetails) + assert.Equal(t, 70_000, result.Logs[0].TokenUsageParsed.PromptTokensDetails.CachedReadTokens) +} + +// TestHybrid_SearchLogsForBillingReportsFetchFailureAsUnpriceable is the important +// difference from hydrateLog, which degrades gracefully on a failed fetch. Graceful +// degradation is right for rendering a message preview and wrong for money: it would +// leave the stub in place and bill it. A lost object must surface as unpriceable. +func TestHybrid_SearchLogsForBillingReportsFetchFailureAsUnpriceable(t *testing.T) { + hybrid, inner, objStore := newTestHybrid(t) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + seedOffloadedBillingLog(t, hybrid, inner, "bill-lost-1", false) + require.Equal(t, 1, objStore.Len()) + + // Simulate an object that is gone (lifecycle rule, bucket migration, corruption). + for _, key := range objStore.Keys() { + require.NoError(t, objStore.Delete(ctx, key)) + } + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err, "one unreadable object must not fail the whole chunk") + assert.Equal(t, []string{"bill-lost-1"}, hydration.Unpriceable) + assert.Empty(t, hydration.Hydrated, "a failed fetch recovered nothing, so there is nothing to back up") + assert.True(t, result.Logs[0].IsUsageDegraded(), + "the row must remain flagged so calculateCostForLog refuses it even if the ID list is ignored") +} + +// TestHybrid_SearchLogsForBillingLeavesNonOffloadedRowsAlone checks we do not pay for +// object fetches we do not need. +func TestHybrid_SearchLogsForBillingLeavesNonOffloadedRowsAlone(t *testing.T) { + hybrid, inner, _ := newTestHybrid(t) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + // Write straight through the inner store so nothing is offloaded and has_object + // stays false. + entry := &Log{ + ID: "bill-inline-1", + Timestamp: time.Now().UTC(), + Provider: "anthropic", + Model: "claude-sonnet-4-20250514", + Status: "success", + Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, inner.CreateIfNotExists(ctx, entry)) + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + require.Empty(t, hydration.Unpriceable) + assert.False(t, result.Logs[0].IsUsageDegraded()) + require.NotNil(t, result.Logs[0].TokenUsageParsed.PromptTokensDetails) + assert.Equal(t, 70_000, result.Logs[0].TokenUsageParsed.PromptTokensDetails.CachedReadTokens) +} + +// TestHybrid_HydrateBillingChunkSkipsRowsThatNeedNothing pins the skip guard. +// +// A deployment can keep token_usage and cache_debug DB-resident via +// object_storage_exclude_fields. Those rows still have has_object set — their message +// history is offloaded — but they already carry every pricing input, so fetching the +// object would be pure waste. Deleting the object proves no fetch is attempted. +func TestHybrid_HydrateBillingChunkSkipsRowsThatNeedNothing(t *testing.T) { + hybrid, inner, objStore := newTestHybridWithExclude(t, []string{"token_usage", "cache_debug"}) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "skip-1", + Timestamp: time.Now().UTC(), + Provider: "anthropic", + Model: "claude-sonnet-4-20250514", + Status: "success", + Object: "chat.completion", + TokenUsageParsed: billingTestUsage(), + CacheDebugParsed: &schemas.BifrostCacheDebug{CacheHit: false}, + InputHistoryParsed: []schemas.ChatMessage{ + {Role: schemas.ChatMessageRoleUser, Content: &schemas.ChatMessageContent{ContentStr: new("a long prompt")}}, + }, + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "skip-1") + + dbRow, err := inner.FindByID(ctx, "skip-1") + require.NoError(t, err) + require.True(t, dbRow.HasObject, "the message history should still have been offloaded") + require.NotEmpty(t, dbRow.TokenUsage, "the exclusion list should have kept token_usage in the DB") + + // Remove the object entirely. A fetch would now fail and mark the row unpriceable, + // so an empty result proves the guard skipped it. + for _, key := range objStore.Keys() { + require.NoError(t, objStore.Delete(ctx, key)) + } + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Hydrated, + "a row that already carries its pricing inputs must not be fetched from object storage") + assert.Empty(t, hydration.Unpriceable, + "and skipping the fetch must not be mistaken for a failed one") + assert.False(t, result.Logs[0].IsUsageDegraded()) + require.NotNil(t, result.Logs[0].TokenUsageParsed.PromptTokensDetails) + assert.Equal(t, 10_000, result.Logs[0].TokenUsageParsed.PromptTokensDetails.CachedWriteTokens) +} + +// TestHybrid_HydrateBillingChunkKeepsModalityRowWithoutTokenUsage pins that "carries no +// token_usage" is not the same as "unpriceable". +// +// A TTS row prices off speech_output.usage.input_chars and never reads token_usage at all +// — computeSpeechCost takes the character count, and computeOCRCost takes no usage +// argument whatsoever. Once the modality payload is back the row has every input pricing +// reads, so reporting it Unpriceable would claim the inputs are unrecoverable when +// hydration in fact fully succeeded, and would leave a real TTS charge unbilled forever. +func TestHybrid_HydrateBillingChunkKeepsModalityRowWithoutTokenUsage(t *testing.T) { + hybrid, inner, objStore := newTestHybrid(t) + defer hybrid.Close(context.Background()) + ctx := context.Background() + + entry := &Log{ + ID: "speech-nousage-1", + Timestamp: time.Now().UTC(), + Provider: "openai", + Model: "gpt-4o-mini-tts", + Status: "success", + Object: "speech", + // Deliberately no TokenUsageParsed: a TTS response carries no token usage, which + // is exactly the shape the unconditional token_usage guard used to reject. + SpeechOutputParsed: &schemas.BifrostSpeechResponse{ + Usage: &schemas.SpeechUsage{InputChars: 512}, + }, + } + require.NoError(t, entry.SerializeFields()) + require.NoError(t, hybrid.CreateIfNotExists(ctx, entry)) + waitForOffload(t, inner, "speech-nousage-1") + require.Equal(t, 1, objStore.Len()) + + // Precondition: the payload really did leave the row, so the hydration below is doing + // the work rather than the column just happening to still be there. + dbLog, err := inner.FindByID(ctx, "speech-nousage-1") + require.NoError(t, err) + require.Empty(t, dbLog.SpeechOutput, "speech_output should be offloaded to object storage") + + result, err := hybrid.SearchLogsForBilling(ctx, SearchFilters{}, PaginationOptions{Limit: 10}) + require.NoError(t, err) + require.Len(t, result.Logs, 1) + + hydration, err := hybrid.HydrateBillingChunk(ctx, []*Log{&result.Logs[0]}) + require.NoError(t, err) + assert.Empty(t, hydration.Unpriceable, + "the row carries every input pricing reads, so it must not be reported unrecoverable") + assert.Equal(t, []string{"speech-nousage-1"}, hydration.Hydrated) + require.NotNil(t, result.Logs[0].SpeechOutputParsed, + "per-character TTS billing needs the recovered speech_output") + assert.Equal(t, 512, result.Logs[0].SpeechOutputParsed.Usage.InputChars) +} diff --git a/framework/logstore/migrations.go b/framework/logstore/migrations.go index b69aafdb7ff..428cc7ad0f6 100644 --- a/framework/logstore/migrations.go +++ b/framework/logstore/migrations.go @@ -280,6 +280,7 @@ var logstoreMigrationSteps = []migrationStep{ {IDs: []string{"webhook_deliveries_add_request_id_column"}, run: migrationAddWebhookDeliveryRequestIDColumn}, {IDs: []string{"logs_add_content_hidden_column"}, run: migrationAddContentHiddenColumn}, {IDs: []string{"logs_add_server_side_fallback_model_column"}, run: migrationAddServerSideFallbackModelColumn}, + {IDs: []string{"logs_add_billing_fidelity_columns"}, run: migrationAddBillingFidelityColumns}, } // areThereAnyPendingMigrations returns true if there are any pending migrations to be applied. @@ -3793,3 +3794,58 @@ func migrationAddServerSideFallbackModelColumn(ctx context.Context, db *gorm.DB, } return nil } + +// migrationAddBillingFidelityColumns adds the served-tier columns cost recomputation +// needs in order to reprice a row at the rates it was actually served at: +// service_tier, speed, and inference_geo. +// +// These need real columns because no existing JSON column can carry them: +// BifrostLLMUsage tags Speed and InferenceGeo `json:"-"` so they never enter any +// serialized usage payload, and service_tier lives on the response envelope rather than +// on usage at all. Keeping them outside payloadFields also means they survive hybrid +// object-storage offload and content-hidden rows, both of which blank token_usage. +// +// There is deliberately no backfill and no denormalized cache-token column. +// +// The tier columns are not backfillable even in principle — the information was never +// captured, so pre-migration rows keep repricing at standard rates, which is the honest +// outcome. And the cache breakdown does not need a column at all: pricing reads it from +// token_usage (hydrated from object storage when offloaded), and a row whose payload +// cannot be recovered is flagged by IsUsageDegraded and skipped rather than priced from +// a lossy fallback. cached_read_tokens exists for an unrelated reason — the matviews and +// token histograms SUM it — and nothing aggregates cache writes, so a matching column +// would have had no consumer while requiring an effectively quadratic backfill to be +// useful on existing rows. +func migrationAddBillingFidelityColumns(ctx context.Context, db *gorm.DB, logger schemas.Logger) error { + migrationName := "logs_add_billing_fidelity_columns" + logger.Info("[logstore] starting migration %s", migrationName) + defer logger.Info("[logstore] finished migration %s", migrationName) + columns := []string{"service_tier", "speed", "inference_geo"} + opts := *migrator.DefaultOptions + opts.UseTransaction = true + m := migrator.New(db, &opts, []*migrator.Migration{{ + ID: migrationName, + Migrate: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, column := range columns { + if err := addColumnIfNotExists(tx, logger, &Log{}, column); err != nil { + return fmt.Errorf("failed to add %s column: %w", column, err) + } + } + return nil + }, + Rollback: func(tx *gorm.DB) error { + tx = tx.WithContext(ctx) + for _, column := range columns { + if err := dropColumnIfExists(tx, logger, &Log{}, column); err != nil { + return err + } + } + return nil + }, + }}) + if err := m.Migrate(); err != nil { + return fmt.Errorf("error while adding billing fidelity columns: %s", err.Error()) + } + return nil +} diff --git a/framework/logstore/payload.go b/framework/logstore/payload.go index bb63971f09f..c434bcb3d47 100644 --- a/framework/logstore/payload.go +++ b/framework/logstore/payload.go @@ -107,6 +107,56 @@ func ExtractPayload(l *Log) map[string]string { // GORM's BeforeCreate/SerializeFields from re-populating TEXT columns. // After calling this, the struct only contains index-weight data suitable // for a lightweight DB INSERT. +// BillingHydrationChunkSize is how many rows a cost recomputation may hydrate at +// once. Kept deliberately tiny: a hydrated row holds its whole offloaded payload, +// which can include full message histories and raw request/response bodies, so this +// is the knob that bounds a recompute worker's peak memory. Billing query pages are +// capped to the same size because DB-resident modality payloads are materialized by +// the query before object-store hydration begins. +const BillingHydrationChunkSize = 3 + +// BillingHydrationResult reports what one hydration pass actually did, so the caller +// does not have to infer it from the rows. +type BillingHydrationResult struct { + // Hydrated lists the rows whose pricing inputs were fetched from object storage. + // These are the only rows worth persisting via BulkBackfillBillingPayloads: every + // other row's inputs already came from the database. + Hydrated []string + // Unpriceable lists rows whose inputs could not be recovered — for example, a + // missing object or unavailable object storage. Callers must skip these rather + // than price them from the lossy fallback. + Unpriceable []string +} + +// BillingPayloadBackfill carries pricing inputs recovered from object storage that are +// worth writing back into the row. +// +// Deliberately only the two small ones. token_usage is a few hundred bytes of counters +// and cache_debug a handful of cache-hit fields, so keeping them in the row is cheap and +// repairs legacy rows written before pricing metadata became unconditionally +// DB-resident. The modality payloads +// (image_generation_output above all, which carries base64 image bytes) are exactly what +// offloading exists to keep out of the database, so they stay in object storage and +// modality rows keep fetching. +type BillingPayloadBackfill struct { + TokenUsage string + CacheDebug string +} + +// ReleaseBillingPayloads drops the payload fields from rows that have already been +// priced, so a batch never accumulates more than one chunk of them. +// +// Safe because pricing is the last thing that reads these: the recalc job keeps only +// the ID, timestamp and computed cost afterwards, and the rows are never written back +// (BulkUpdateCost takes an id → cost map). +func ReleaseBillingPayloads(logs []*Log) { + for _, l := range logs { + if l != nil { + ClearPayload(l) + } + } +} + func ClearPayload(l *Log) { // Clear serialized TEXT columns. l.InputHistory = "" diff --git a/framework/logstore/rdb.go b/framework/logstore/rdb.go index 3b760026ed0..5745987bd12 100644 --- a/framework/logstore/rdb.go +++ b/framework/logstore/rdb.go @@ -700,6 +700,88 @@ func (s *RDBLogStore) bulkUpdateCostPostgres(ctx context.Context, updates map[st // SearchLogs searches for logs in the database without calculating statistics. func (s *RDBLogStore) SearchLogs(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) { + return s.searchLogs(ctx, filters, pagination, s.listSelectColumns()) +} + +// SearchLogsForBilling searches with the billing projection. +func (s *RDBLogStore) SearchLogsForBilling(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) { + result, err := s.searchLogs(ctx, filters, pagination, s.billingSelectColumns()) + if err != nil || result == nil { + return result, err + } + for i := range result.Logs { + stripNonBillingPayloadBytes(&result.Logs[i]) + } + return result, nil +} + +// HydrateBillingChunk is a no-op for a pure RDB store: nothing is offloaded, so every +// pricing input is already in the row. The hybrid store overrides this to fetch. +func (s *RDBLogStore) HydrateBillingChunk(_ context.Context, _ []*Log) (BillingHydrationResult, error) { + return BillingHydrationResult{}, nil +} + +// BulkBackfillBillingPayloads writes recovered token_usage / cache_debug back into their +// rows so later recomputes need no object fetch. +// +// Only ever called with rows whose payload was actually fetched, which in a pure RDB +// store is never — this exists for the hybrid store layered on top, which delegates here. +func (s *RDBLogStore) BulkBackfillBillingPayloads(ctx context.Context, updates map[string]BillingPayloadBackfill) error { + if len(updates) == 0 { + return nil + } + + // ClickHouse has no row UPDATE; mutations are async and expensive, which is why + // BulkUpdateCost re-inserts the whole row there instead. Re-inserting just to + // populate a cache of something object storage already holds is not worth that + // cost, so ClickHouse deployments keep fetching. Skipped rather than errored + // because the backfill is an optimisation, not part of the job's output. + if s.db.Dialector.Name() == "clickhouse" { + return nil + } + + return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + for id, backfill := range updates { + // Both written unconditionally, including an empty cache_debug: the gate + // infers "already backfilled" from token_usage being present on an + // offloaded row, so an empty cache_debug beside it correctly reads as + // "the object held none" rather than "not yet recovered". + columns := map[string]any{ + "token_usage": backfill.TokenUsage, + "cache_debug": backfill.CacheDebug, + } + if err := tx.Model(&Log{}).Where("id = ?", id).Updates(columns).Error; err != nil { + return err + } + } + return nil + }) +} + +// stripNonBillingPayloadBytes releases bytes that a modality payload carries but +// pricing never reads. +// +// Today that is generated image data. computeImageCost needs the image *count* plus +// size and quality — extractCostInput passes len(Data) to populateOutputImageCount and +// never touches the images themselves — while ImageData.B64JSON holds full base64 +// bytes, megabytes per image. A recompute holds a whole batch at once, so keeping them +// is a straight path to an OOM on an image-generation window. +// +// The serialized column is cleared too: it has already been parsed, and the row is +// never re-deserialized or written back. +func stripNonBillingPayloadBytes(l *Log) { + if l == nil || l.ImageGenerationOutputParsed == nil { + return + } + for i := range l.ImageGenerationOutputParsed.Data { + l.ImageGenerationOutputParsed.Data[i].B64JSON = "" + l.ImageGenerationOutputParsed.Data[i].URL = "" + l.ImageGenerationOutputParsed.Data[i].RevisedPrompt = "" + } + l.ImageGenerationOutput = "" +} + +func (s *RDBLogStore) searchLogs(ctx context.Context, filters SearchFilters, pagination PaginationOptions, selectColumns string) (*SearchResult, error) { // Build order clause up front (needed by the data goroutine). direction := "DESC" if pagination.Order == "asc" { @@ -755,7 +837,7 @@ func (s *RDBLogStore) SearchLogs(ctx context.Context, filters SearchFilters, pag g.Go(func() error { dataQuery := s.ScopedDB(gCtx).Model(&Log{}) dataQuery = s.applyFilters(dataQuery, filters) - dataQuery = dataQuery.Order(orderClause).Select(s.listSelectColumns()).Limit(limit) + dataQuery = dataQuery.Order(orderClause).Select(selectColumns).Limit(limit) if pagination.Offset > 0 { dataQuery = dataQuery.Offset(pagination.Offset) } @@ -971,6 +1053,9 @@ func (s *RDBLogStore) listSelectColumns() string { "metadata", "cache_debug", "is_large_payload_request", "is_large_payload_response", "prompt_tokens", "completion_tokens", "total_tokens", + "cached_read_tokens", + "has_object", "content_hidden", + "service_tier", "speed", "inference_geo", "created_at", }, ", ") @@ -1022,6 +1107,99 @@ func (s *RDBLogStore) listSelectColumns() string { return baseCols + ", " + inputHistoryExpr + ", " + responsesInputExpr + ", " + outputMessageExpr } +// billingPayloadColumns are the payload columns cost recomputation reads to recover a +// billing dimension that BifrostLLMUsage cannot express: audio duration and the +// audio/text token split, TTS input character counts, output image counts with size +// and quality, video duration, and OCR pages processed. Without them those request +// types are priced off aggregate token counts alone, which is wrong for every one. +// +// Each is keyed to the object_type values that actually bill on it, so a chat row +// never drags an unrelated payload into memory — see billingSelectColumns. +var billingPayloadColumns = map[string][]string{ + "speech_output": {"speech", "speech_stream"}, + "transcription_output": {"transcription", "transcription_stream"}, + "image_generation_output": { + "image_generation", "image_generation_stream", + "image_edit", "image_edit_stream", + "image_variation", + }, + "video_generation_output": {"video_generation", "video_remix"}, + "ocr_output": {"ocr"}, +} + +// billingScalarColumns is every non-payload column cost recomputation reads, and +// nothing else. Traced from calculateCostForLog, pricingScopesForLog, +// servedTierFromLog, isKnownZeroCostLog, and the recalc job's cursor bookkeeping. +// +// Deliberately NOT built on listSelectColumns: that projection serves /api/logs and +// carries message previews, content summaries, metadata and every denormalized +// governance name — none of which pricing looks at. A recompute holds a whole batch in +// memory at once, so every column it does not need is pure ballast. +var billingScalarColumns = []string{ + // Identity and the object-storage key. + "id", "timestamp", "object_type", + // Pricing lookup: provider, the wire/alias/canonical model triplet, and the + // server-side fallback model that resolvePricing ranks first. + "provider", "model", "alias", "canonical_model_name", "server_side_fallback_model", + // Override scopes. + "selected_key_id", "virtual_key_id", "user_id", + // Usage, and the denormalized fallback used when token_usage was offloaded. + "token_usage", "prompt_tokens", "completion_tokens", "total_tokens", "cached_read_tokens", + // Semantic-cache billing. + "cache_debug", + // Whether the payload was offloaded, and whether it can ever be fetched back. + "has_object", "content_hidden", + // Served tier: scales every token rate. + "service_tier", "speed", "inference_geo", +} + +// billingSelectColumns returns the SELECT clause for cost recomputation. +// +// The modality payload columns are wrapped in a CASE on object_type so each row pulls +// only the payload its own request type bills on. This matters most for +// image_generation_output, which serializes ImageData.B64JSON — full base64 image bytes, +// megabytes per image. Selecting it unconditionally would pull that for every row in +// the batch including plain chat rows, and pricing only ever reads len(Data). +// The same CASE pattern guards output_message in listSelectColumns for realtime turns. +func (s *RDBLogStore) billingSelectColumns() string { + cols := make([]string, 0, len(billingScalarColumns)+len(billingPayloadColumns)) + cols = append(cols, billingScalarColumns...) + + // Sorted so the generated SQL is stable across calls (map iteration is not). + payloadCols := make([]string, 0, len(billingPayloadColumns)) + for col := range billingPayloadColumns { + payloadCols = append(payloadCols, col) + } + sort.Strings(payloadCols) + + for _, col := range payloadCols { + objectTypes := billingPayloadColumns[col] + quoted := make([]string, 0, len(objectTypes)) + for _, t := range objectTypes { + quoted = append(quoted, "'"+t+"'") + } + cols = append(cols, fmt.Sprintf( + "CASE WHEN object_type IN (%s) THEN %s ELSE NULL END AS %s", + strings.Join(quoted, ", "), col, col, + )) + } + return strings.Join(cols, ", ") +} + +// billingPayloadColumnFor returns the payload column an object_type bills on, or "" +// when the request type prices purely off token counts. Used both to gate the +// projection and to decide whether a row still needs an object fetch. +func billingPayloadColumnFor(objectType string) string { + for col, objectTypes := range billingPayloadColumns { + for _, t := range objectTypes { + if t == objectType { + return col + } + } + } + return "" +} + // GetStats calculates statistics for logs matching the given filters. func (s *RDBLogStore) GetStats(ctx context.Context, filters SearchFilters) (*SearchStats, error) { // Stats has a stricter matview gate than other paths: short windows go to diff --git a/framework/logstore/store.go b/framework/logstore/store.go index 51643efbf2e..d19cf858ace 100644 --- a/framework/logstore/store.go +++ b/framework/logstore/store.go @@ -32,6 +32,40 @@ type LogStore interface { FindAllDistinct(ctx context.Context, query any, fields ...string) ([]*Log, error) HasLogs(ctx context.Context) (bool, error) SearchLogs(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) + // SearchLogsForBilling is SearchLogs projected for cost recomputation: exactly the + // columns pricing reads and nothing else, with each modality payload pulled only + // for the request types that bill on it. + // + // It does NOT hydrate offloaded payloads — that would put every row's payload in + // memory at once. Callers walk the result in BillingHydrationChunkSize slices, + // calling HydrateBillingChunk and then ReleaseBillingPayloads on each. + SearchLogsForBilling(ctx context.Context, filters SearchFilters, pagination PaginationOptions) (*SearchResult, error) + // HydrateBillingChunk restores offloaded pricing inputs for a small slice of rows. + // It skips rows that already carry everything pricing needs, so a store with + // nothing offloaded performs no I/O at all. + // + // BillingHydrationResult.Unpriceable carries the IDs whose inputs could not be + // recovered — for example, a missing object or unavailable object storage. Callers + // must skip those rather than price them: the + // denormalized fallback usage omits the cache breakdown, and pricing it charges + // every cached token at the full input rate. Hydrated carries the IDs actually + // fetched, which are the only rows worth passing to BulkBackfillBillingPayloads. + // + // Pass at most BillingHydrationChunkSize rows and release them before requesting + // more; the whole point is to bound how many payloads are resident at once. + HydrateBillingChunk(ctx context.Context, logs []*Log) (BillingHydrationResult, error) + // BulkBackfillBillingPayloads writes pricing inputs recovered from object storage + // back into their rows, so a later recompute reads them from the database instead + // of fetching the object again. + // + // This makes recompute self-healing: the first pass over a window pays the object + // fetches, and every pass after it is served entirely from the DB. It is + // opportunistic by design — only rows that were actually fetched get backfilled, so + // there is no table-wide migration and no scan. + // + // Callers should treat a failure as non-fatal: the cost update is the job's real + // output, and a missed backfill only means the next run fetches again. + BulkBackfillBillingPayloads(ctx context.Context, updates map[string]BillingPayloadBackfill) error GetSessionLogs(ctx context.Context, sessionID string, pagination PaginationOptions) (*SessionDetailResult, error) GetSessionSummary(ctx context.Context, sessionID string) (*SessionSummaryResult, error) GetStats(ctx context.Context, filters SearchFilters) (*SearchStats, error) diff --git a/framework/logstore/tables.go b/framework/logstore/tables.go index d2cc69b6d43..e0733183344 100644 --- a/framework/logstore/tables.go +++ b/framework/logstore/tables.go @@ -241,12 +241,29 @@ type Log struct { BudgetIDs *string `gorm:"type:text" json:"-"` // JSON serialized []string of budget IDs applicable to this request RateLimitIDs *string `gorm:"type:text" json:"-"` // JSON serialized []string of rate limit IDs applicable to this request - // Denormalized token fields for easier querying + // Denormalized token fields for easier querying. cached_read_tokens earns its + // place because the matviews and token histograms SUM it (see matviews.go and + // GetTokenHistogram). There is deliberately no cache-write counterpart: nothing + // aggregates cache writes, and pricing reads the cache breakdown out of + // token_usage rather than from a column, so it would have had no consumer. PromptTokens int `gorm:"default:0" json:"-"` CompletionTokens int `gorm:"default:0" json:"-"` TotalTokens int `gorm:"index:idx_logs_total_tokens;default:0" json:"-"` CachedReadTokens int `gorm:"default:0" json:"-"` + // Served billing tier, denormalized so cost recomputation can reprice a row + // at the rates it was actually served at. These are deliberately NOT payload + // fields (see payload.go): they must survive hybrid object-storage offload + // and content-hidden rows, both of which blank the token_usage column. + // + // token_usage cannot carry them — BifrostLLMUsage tags Speed and InferenceGeo + // `json:"-"`, so they never appear in any serialized usage payload. Without + // these columns tierFromResponse sees nothing and reprices flex traffic at + // standard rates. + ServiceTier *string `gorm:"type:varchar(32)" json:"service_tier,omitempty"` // OpenAI served tier: "priority" / "flex" / "default" + Speed *string `gorm:"type:varchar(32)" json:"speed,omitempty"` // Anthropic served speed: "fast" / "standard" + InferenceGeo *string `gorm:"type:varchar(32)" json:"inference_geo,omitempty"` // Anthropic data residency, e.g. "us" + CreatedAt time.Time `gorm:"index;not null" json:"created_at"` // Virtual fields for JSON output - these will be populated when needed @@ -295,6 +312,32 @@ type Log struct { VirtualKey *tables.TableVirtualKey `gorm:"-" json:"virtual_key,omitempty"` // redacted SelectedKey *schemas.Key `gorm:"-" json:"selected_key,omitempty"` // redacted RoutingRule *tables.TableRoutingRule `gorm:"-" json:"routing_rule,omitempty"` // redacted + + // usageRebuiltFromColumns records that TokenUsageParsed came from the lossy + // denormalized-column rebuild in DeserializeFields rather than from a real + // token_usage payload. Unexported so it is invisible to GORM and to JSON: it + // is provenance for the current read, not row state. Read via IsUsageDegraded. + usageRebuiltFromColumns bool + + // billingPayloadsHydrated records that this row's offloaded payload has already + // been fetched for billing. Needed because "absent" and "was never written" look + // identical in a column: a row with no cache_debug still has an empty cache_debug + // after a successful fetch, and without this flag the gate would fetch it again + // every time. Same provenance-not-state reasoning as above. + billingPayloadsHydrated bool +} + +// IsUsageDegraded reports whether TokenUsageParsed is the lossy stub rebuilt +// from denormalized columns instead of the real token_usage payload. +// +// It exists for billing. The stub carries only prompt/completion/total plus the +// cached read/write totals, and PromptTokens is inclusive of the cache buckets — +// so pricing a stub charges every cached token at the full input rate and can +// inflate a cache-heavy request several fold. Callers that compute money must +// skip these rows rather than price them; callers that render tokens (the log +// list, the UI) are free to use the stub, which is what it was built for. +func (l *Log) IsUsageDegraded() bool { + return l != nil && l.usageRebuiltFromColumns } // NewLogEntryFromMap creates a new Log from a map[string]interface{} @@ -749,9 +792,21 @@ func (l *Log) DeserializeFields() error { } if l.TokenUsage != "" { + // Reset before parsing so the result reflects only what the payload says. + // DeserializeFields runs twice on the hybrid hydration path, and unmarshalling + // over a retained stub would let a zero-valued (omitempty-elided) field in the + // payload inherit the stub's number instead of zero. + l.TokenUsageParsed = nil if err := sonic.Unmarshal([]byte(l.TokenUsage), &l.TokenUsageParsed); err != nil { // Log error but don't fail the operation - initialize as nil l.TokenUsageParsed = nil + } else { + // A real payload supersedes any earlier column rebuild. This matters on + // the hybrid hydration path: AfterFind builds the stub while token_usage + // is still blank, then hydration fills the column and re-deserializes. + // Without clearing the flag the row would stay marked degraded and + // billing would skip a row it can now price correctly. + l.usageRebuiltFromColumns = false } } @@ -944,9 +999,16 @@ func (l *Log) DeserializeFields() error { // Hybrid log store offloads token_usage to object storage but keeps denormalized // prompt/completion/total/cached columns in the DB for analytics. Rebuild the virtual // field so list APIs and the UI can render tokens without hydrating from S3 — - // same role content_summary plays for message previews. Only the cached-read detail - // is denormalized; richer details (e.g. completion_tokens_details) live solely in the - // offloaded payload and are restored on detail reads that hydrate from object storage. + // same role content_summary plays for message previews. Only the cached-read total + // is denormalized; the cache-write total, the 5m/1h split, and richer details such + // as completion_tokens_details live solely in the offloaded payload and are restored + // on detail reads that hydrate from object storage. + // + // This rebuild is LOSSY and must never be used for billing. PromptTokens is + // inclusive of the cache buckets, so a consumer that prices this stub without + // the details reprices every cached token at the full input rate. Billing reads + // go through SearchLogsForBilling, which hydrates the real payload; see + // IsUsageDegraded for the check that keeps the recalc job from pricing a stub. if l.TokenUsage == "" && l.TokenUsageParsed == nil && (l.PromptTokens != 0 || l.CompletionTokens != 0 || l.TotalTokens != 0) { usage := &schemas.BifrostLLMUsage{ PromptTokens: l.PromptTokens, @@ -959,6 +1021,7 @@ func (l *Log) DeserializeFields() error { } } l.TokenUsageParsed = usage + l.usageRebuiltFromColumns = true } return nil diff --git a/framework/streaming/accumulator.go b/framework/streaming/accumulator.go index 5a21bc8607d..e1910bd73c7 100644 --- a/framework/streaming/accumulator.go +++ b/framework/streaming/accumulator.go @@ -54,6 +54,7 @@ func (a *Accumulator) putChatStreamChunk(chunk *ChatStreamChunk) { chunk.ErrorDetails = nil chunk.FinishReason = nil chunk.TokenUsage = nil + chunk.ServiceTier = nil chunk.RawResponse = nil a.chatStreamChunkPool.Put(chunk) } @@ -108,6 +109,7 @@ func (a *Accumulator) putResponsesStreamChunk(chunk *ResponsesStreamChunk) { chunk.ErrorDetails = nil chunk.FinishReason = nil chunk.TokenUsage = nil + chunk.ServiceTier = nil chunk.RawResponse = nil a.responsesStreamChunkPool.Put(chunk) } diff --git a/framework/streaming/accumulator_test.go b/framework/streaming/accumulator_test.go index f82fe2408cd..3ca11e74910 100644 --- a/framework/streaming/accumulator_test.go +++ b/framework/streaming/accumulator_test.go @@ -94,6 +94,36 @@ func TestChatStreamingFinalChunkNoDeadlock(t *testing.T) { } } +func TestAccumulatedChatStreamPreservesServiceTierBeforeUsageOnlyChunk(t *testing.T) { + accumulator := NewAccumulator(nil, bifrost.NewDefaultLogger(schemas.LogLevelError)) + t.Cleanup(accumulator.Cleanup) + + requestID := "chat-service-tier" + priority := schemas.BifrostServiceTierPriority + requireNoError := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + requireNoError(accumulator.addChatStreamChunk(requestID, StreamTypeChat, &ChatStreamChunk{ + ChunkIndex: 1, + Timestamp: time.Now(), + ServiceTier: &priority, + }, false)) + requireNoError(accumulator.addChatStreamChunk(requestID, StreamTypeChat, &ChatStreamChunk{ + ChunkIndex: 2, + Timestamp: time.Now(), + TokenUsage: &schemas.BifrostLLMUsage{TotalTokens: 1}, + }, true)) + + data, err := accumulator.processAccumulatedChatStreamingChunks(requestID, nil, true) + requireNoError(err) + if data.ServiceTier == nil || *data.ServiceTier != schemas.BifrostServiceTierPriority { + t.Fatalf("expected priority service tier, got %v", data.ServiceTier) + } +} + // TestResponsesStreamingFinalChunkNoDeadlock tests Responses streaming doesn't deadlock func TestResponsesStreamingFinalChunkNoDeadlock(t *testing.T) { logger := bifrost.NewDefaultLogger(schemas.LogLevelDebug) diff --git a/framework/streaming/chat.go b/framework/streaming/chat.go index b087507eeec..2a79595f3de 100644 --- a/framework/streaming/chat.go +++ b/framework/streaming/chat.go @@ -461,6 +461,16 @@ func (a *Accumulator) processAccumulatedChatStreamingChunks(requestID string, re } data.FinishReason = lastChunk.FinishReason } + // service_tier can arrive before a trailing usage-only or synthetic terminal + // chunk, so retain the newest non-nil value rather than reading only the + // highest-index chunk. + tierChunkIndex := -1 + for _, streamChunk := range accumulator.ChatStreamChunks { + if streamChunk.ServiceTier != nil && streamChunk.ChunkIndex > tierChunkIndex { + data.ServiceTier = streamChunk.ServiceTier + tierChunkIndex = streamChunk.ChunkIndex + } + } // The highest-index chunk can carry a nil finish_reason (a usage-only chunk, // or the synthetic terminal chunk the OpenAI-compatible handler appends after // forwarding finish_reason on a content chunk). Fall back to the newest chunk @@ -575,6 +585,9 @@ func (a *Accumulator) processChatStreamingResponse(ctx *schemas.BifrostContext, if result.ChatResponse.Usage != nil && result.ChatResponse.Usage.TotalTokens > 0 { chunk.TokenUsage = result.ChatResponse.Usage } + if result.ChatResponse.ServiceTier != nil { + chunk.ServiceTier = new(schemas.BifrostServiceTier(*result.ChatResponse.ServiceTier)) + } chunk.ChunkIndex = result.ChatResponse.ExtraFields.ChunkIndex if result.ChatResponse.ExtraFields.RawResponse != nil { chunk.RawResponse = bifrost.Ptr(fmt.Sprintf("%v", result.ChatResponse.ExtraFields.RawResponse)) diff --git a/framework/streaming/responses.go b/framework/streaming/responses.go index c0421fe0773..0651cb1be17 100644 --- a/framework/streaming/responses.go +++ b/framework/streaming/responses.go @@ -989,6 +989,15 @@ func (a *Accumulator) processAccumulatedResponsesStreamingChunks(requestID strin } data.FinishReason = lastChunk.FinishReason } + // The response envelope carrying service_tier can precede a later usage-only + // event, so retain the newest non-nil tier across the stream. + tierChunkIndex := -1 + for _, streamChunk := range accumulator.ResponsesStreamChunks { + if streamChunk.ServiceTier != nil && streamChunk.ChunkIndex > tierChunkIndex { + data.ServiceTier = streamChunk.ServiceTier + tierChunkIndex = streamChunk.ChunkIndex + } + } // Accumulate raw response using strings.Builder to avoid O(n^2) string concatenation if len(accumulator.ResponsesStreamChunks) > 0 { @@ -1054,6 +1063,10 @@ func (a *Accumulator) processResponsesStreamingResponse(ctx *schemas.BifrostCont result.ResponsesStreamResponse.Response.Usage != nil { chunk.TokenUsage = result.ResponsesStreamResponse.Response.Usage.ToBifrostLLMUsage() } + if result.ResponsesStreamResponse.Response != nil && + result.ResponsesStreamResponse.Response.ServiceTier != nil { + chunk.ServiceTier = new(schemas.BifrostServiceTier(*result.ResponsesStreamResponse.Response.ServiceTier)) + } chunk.ChunkIndex = result.ResponsesStreamResponse.ExtraFields.ChunkIndex if isFinalChunk { if a.pricingManager != nil { diff --git a/framework/streaming/responses_test.go b/framework/streaming/responses_test.go index cae49cc7508..14be278810c 100644 --- a/framework/streaming/responses_test.go +++ b/framework/streaming/responses_test.go @@ -8,6 +8,7 @@ import ( bifrost "github.com/maximhq/bifrost/core" "github.com/maximhq/bifrost/core/schemas" + "github.com/stretchr/testify/require" ) func testResponsesAccumulator(tb testing.TB) *Accumulator { @@ -17,6 +18,28 @@ func testResponsesAccumulator(tb testing.TB) *Accumulator { return acc } +func TestAccumulatedResponsesStreamPreservesServiceTierBeforeUsageOnlyChunk(t *testing.T) { + acc := testResponsesAccumulator(t) + requestID := "responses-service-tier" + flex := schemas.BifrostServiceTierFlex + + require.NoError(t, acc.addResponsesStreamChunk(requestID, &ResponsesStreamChunk{ + ChunkIndex: 1, + Timestamp: time.Now(), + ServiceTier: &flex, + }, false)) + require.NoError(t, acc.addResponsesStreamChunk(requestID, &ResponsesStreamChunk{ + ChunkIndex: 2, + Timestamp: time.Now(), + TokenUsage: &schemas.BifrostLLMUsage{TotalTokens: 1}, + }, true)) + + data, err := acc.processAccumulatedResponsesStreamingChunks(requestID, nil, true) + require.NoError(t, err) + require.NotNil(t, data.ServiceTier) + require.Equal(t, schemas.BifrostServiceTierFlex, *data.ServiceTier) +} + // TestBuildResponsesMessageConcatenatesTextDeltas verifies that many streamed // text deltas are joined in order. This is the path that previously accumulated // via O(n²) `*Text += delta`; the builder rewrite must produce identical bytes. diff --git a/framework/streaming/types.go b/framework/streaming/types.go index e810604aeda..605855875c9 100644 --- a/framework/streaming/types.go +++ b/framework/streaming/types.go @@ -35,6 +35,7 @@ type AccumulatedData struct { ToolCalls []schemas.ChatAssistantMessageToolCall ErrorDetails *schemas.BifrostError TokenUsage *schemas.BifrostLLMUsage + ServiceTier *schemas.BifrostServiceTier CacheDebug *schemas.BifrostCacheDebug Cost *float64 AudioOutput *schemas.BifrostSpeechResponse @@ -79,6 +80,7 @@ type ChatStreamChunk struct { FinishReason *string // If this is the final chunk LogProbs *schemas.BifrostLogProbs // LogProbs if available TokenUsage *schemas.BifrostLLMUsage // Token usage if available + ServiceTier *schemas.BifrostServiceTier // Served OpenAI tier if available SemanticCacheDebug *schemas.BifrostCacheDebug // Semantic cache debug if available Cost *float64 // Cost in dollars from pricing plugin ErrorDetails *schemas.BifrostError // Error if any @@ -92,6 +94,7 @@ type ResponsesStreamChunk struct { StreamResponse *schemas.BifrostResponsesStreamResponse // The actual stream response FinishReason *string // If this is the final chunk TokenUsage *schemas.BifrostLLMUsage // Token usage if available + ServiceTier *schemas.BifrostServiceTier // Served OpenAI tier if available SemanticCacheDebug *schemas.BifrostCacheDebug // Semantic cache debug if available Cost *float64 // Cost in dollars from pricing plugin ErrorDetails *schemas.BifrostError // Error if any diff --git a/plugins/logging/costfidelity_test.go b/plugins/logging/costfidelity_test.go new file mode 100644 index 00000000000..5d8a157c6cd --- /dev/null +++ b/plugins/logging/costfidelity_test.go @@ -0,0 +1,920 @@ +package logging + +import ( + "context" + "errors" + "math" + "testing" + "time" + + "github.com/maximhq/bifrost/core/schemas" + "github.com/maximhq/bifrost/framework/logstore" + "github.com/maximhq/bifrost/framework/modelcatalog" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// These tests pin cost recomputation to the cost computed live at request time. +// +// The bug they guard against: calculateCostForLog rebuilds a BifrostResponse from a +// log row, and every pricing input the row fails to carry silently changes the +// price. The worst case is the cache breakdown — PromptTokens is normalized to be +// inclusive of the cache buckets (see the Anthropic chat mapping) and computeTextCost +// subtracts them back out, so losing the breakdown reprices every cached token at the +// full input rate. On a 70%-cache-read request that is a ~2.5x overbill; at 85% it is +// ~4.3x. A user hit ~3x in production. +// +// Every assertion here compares against live pricing on an equivalent response rather +// than merely checking cost > 0. A `> 0` assertion is exactly what let the drift ship. + +const costEpsilon = 1e-12 + +func assertCostsEqual(t *testing.T, label string, got, want float64) { + t.Helper() + if math.Abs(got-want) > costEpsilon { + t.Fatalf("%s: recomputed cost = %.12f, live cost = %.12f (ratio %.4fx)", label, got, want, got/want) + } +} + +func newCostFidelityPlugin(t *testing.T) *LoggerPlugin { + t.Helper() + return &LoggerPlugin{ + store: newTestStore(t), + pricingManager: newTestPricingManager(t), + logger: testLogger{}, + } +} + +// liveCost prices a response the way the PostHook does at request time. +func liveCost(t *testing.T, plugin *LoggerPlugin, resp *schemas.BifrostResponse, provider string) float64 { + t.Helper() + scopes := modelcatalog.PricingLookupScopes{Provider: provider} + return plugin.pricingManager.CalculateCost(resp, &scopes) +} + +// TestCalculateCostForLogMatchesLiveForCachedRequest is the regression test for the +// reported ~3x inflation. A 100k-token Anthropic prompt that is 70% cache read and +// 10% cache write must reprice to the same number it was billed at live. +func TestCalculateCostForLogMatchesLiveForCachedRequest(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + const ( + promptTokens = 100_000 + cachedReadTokens = 70_000 + cachedWriteTokens = 10_000 + completionTokens = 500 + ) + usage := &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: cachedReadTokens, + CachedWriteTokens: cachedWriteTokens, + }, + } + + want := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{ + Provider: schemas.Anthropic, + Model: "claude-sonnet-4-20250514", + }, + }, + }, + }, string(schemas.Anthropic)) + if want <= 0 { + t.Fatalf("live cost must be positive for the comparison to mean anything, got %v", want) + } + + entry := &logstore.Log{ + ID: "req-cache-parity", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + CachedReadTokens: cachedReadTokens, + TokenUsageParsed: usage, + } + + got, err := plugin.calculateCostForLog(entry) + if err != nil { + t.Fatalf("calculateCostForLog() error = %v", err) + } + assertCostsEqual(t, "cached chat request", got, want) + + // Guard the magnitude of the original defect explicitly: pricing the same token + // count with no cache breakdown must be dramatically more expensive, so this test + // would have failed loudly against the old code rather than passing on a + // coincidence. + noBreakdown := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + }, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{ + Provider: schemas.Anthropic, + Model: "claude-sonnet-4-20250514", + }, + }, + }, + }, string(schemas.Anthropic)) + if noBreakdown <= want*1.5 { + t.Fatalf("expected a cache-blind price to be far higher than the correct one; correct=%.12f blind=%.12f", want, noBreakdown) + } +} + +// TestCalculateCostForLogRefusesDegradedUsage covers the hybrid-store case that +// produced the production overbill: token_usage is offloaded to object storage, so +// DeserializeFields rebuilds a stub from the denormalized columns. Pricing that stub +// is what inflated the cost, so recomputation must refuse it and report an error the +// job can count as unpriceable. +func TestCalculateCostForLogRefusesDegradedUsage(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + // Mimic a row read back from a store that offloaded its payload: the token_usage + // column is blank but the denormalized counters survive. + entry := &logstore.Log{ + ID: "req-degraded", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + TokenUsage: "", + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + } + if err := entry.DeserializeFields(); err != nil { + t.Fatalf("DeserializeFields() error = %v", err) + } + if !entry.IsUsageDegraded() { + t.Fatal("expected the column rebuild to mark usage as degraded") + } + + if _, err := plugin.calculateCostForLog(entry); !errors.Is(err, errPricingInputsUnavailable) { + t.Fatalf("expected errPricingInputsUnavailable, got %v", err) + } +} + +// TestCalculateCostForLogPricesDegradedDirectCacheHitAsZero pins the ordering of the two +// gates in calculateCostForLog. +// +// A direct cache hit was served without an LLM call, so calculateCostWithCache returns +// zero without reading usage at all — the token breakdown is irrelevant to the price. +// Refusing such a row for degraded usage would leave its cost nil forever, and because +// MissingCostOnly matches on cost <= 0 every later pass would fetch and reject it again. +// The zero must be recorded instead, which is terminal. +func TestCalculateCostForLogPricesDegradedDirectCacheHitAsZero(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + // Same offloaded-row shape as TestCalculateCostForLogRefusesDegradedUsage, plus the + // cache_debug column. cache_debug is denormalized in the DB, so it survives on rows + // whose token_usage payload does not. + entry := &logstore.Log{ + ID: "req-degraded-direct-hit", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + TokenUsage: "", + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + CacheDebug: `{"cache_hit":true,"hit_type":"direct"}`, + } + require.NoError(t, entry.DeserializeFields()) + require.True(t, entry.IsUsageDegraded(), + "the fixture must actually be degraded or this test proves nothing") + + cost, err := plugin.calculateCostForLog(entry) + require.NoError(t, err, "a provably-free row must not be reported unpriceable") + assert.Zero(t, cost) +} + +// TestDeserializeFieldsClearsDegradedFlagAfterHydration checks the hybrid hydration +// ordering: AfterFind builds the stub while token_usage is still blank, then hydration +// fills the column and re-deserializes. If the flag survived that, billing would skip +// rows it can now price correctly. +func TestDeserializeFieldsClearsDegradedFlagAfterHydration(t *testing.T) { + entry := &logstore.Log{ + ID: "req-hydrated", + Timestamp: time.Now().UTC(), + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + } + if err := entry.DeserializeFields(); err != nil { + t.Fatalf("first DeserializeFields() error = %v", err) + } + if !entry.IsUsageDegraded() { + t.Fatal("expected degraded usage before hydration") + } + + // Hydration writes the serialized column, then re-deserializes. + entry.TokenUsage = `{"prompt_tokens":100000,"completion_tokens":500,"total_tokens":100500,` + + `"prompt_tokens_details":{"cached_read_tokens":70000,"cached_write_tokens":10000}}` + if err := entry.DeserializeFields(); err != nil { + t.Fatalf("second DeserializeFields() error = %v", err) + } + if entry.IsUsageDegraded() { + t.Fatal("expected the degraded flag to clear once a real payload parsed") + } + if entry.TokenUsageParsed.PromptTokensDetails.CachedWriteTokens != 10_000 { + t.Fatalf("expected hydrated cache-write detail, got %+v", entry.TokenUsageParsed.PromptTokensDetails) + } +} + +// TestCalculateCostForLogPreservesServedTier pins the store-independent tier leak: the +// Log table had no service_tier column and BifrostLLMUsage tags Speed/InferenceGeo +// `json:"-"`, so recomputation priced every row at standard rates. On OpenAI that +// silently moved priority traffic onto the cheaper standard rate. +func TestCalculateCostForLogPreservesServedTier(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + const promptTokens, completionTokens = 1_000, 200 + usage := &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + } + priority := schemas.BifrostServiceTierPriority + + want := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ServiceTier: &priority, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI, Model: "gpt-4o"}, + }, + }, + }, string(schemas.OpenAI)) + + standard := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI, Model: "gpt-4o"}, + }, + }, + }, string(schemas.OpenAI)) + if math.Abs(want-standard) <= costEpsilon { + t.Skip("test datasheet has no distinct priority rate for gpt-4o; tier parity is unobservable") + } + + tier := string(schemas.BifrostServiceTierPriority) + entry := &logstore.Log{ + ID: "req-tier-parity", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.OpenAI), + Model: "gpt-4o", + Status: "success", + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + ServiceTier: &tier, + TokenUsageParsed: usage, + } + + got, err := plugin.calculateCostForLog(entry) + if err != nil { + t.Fatalf("calculateCostForLog() error = %v", err) + } + assertCostsEqual(t, "priority-tier chat request", got, want) +} + +// TestApplyServedTierToEntryRecordsResponseAndUsageSources checks both capture paths. +// Non-streaming responses carry the tier on the response envelope; the streaming +// accumulator has no envelope, so Speed/InferenceGeo must be read off the usage struct +// (which is why the provider populates them there). +func TestApplyServedTierToEntryRecordsResponseAndUsageSources(t *testing.T) { + flex := schemas.BifrostServiceTierFlex + + t.Run("from response envelope", func(t *testing.T) { + entry := &logstore.Log{} + applyServedTierToEntry(entry, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ServiceTier: &flex}, + }, nil) + if entry.ServiceTier == nil || *entry.ServiceTier != "flex" { + t.Fatalf("expected service_tier flex, got %v", entry.ServiceTier) + } + }) + + t.Run("from usage on the streaming path", func(t *testing.T) { + entry := &logstore.Log{} + applyServedTierToEntry(entry, nil, &schemas.BifrostLLMUsage{ + Speed: new("fast"), + InferenceGeo: new("us"), + }) + if entry.Speed == nil || *entry.Speed != "fast" { + t.Fatalf("expected speed fast, got %v", entry.Speed) + } + if entry.InferenceGeo == nil || *entry.InferenceGeo != "us" { + t.Fatalf("expected inference_geo us, got %v", entry.InferenceGeo) + } + }) + + t.Run("a later usage-only chunk does not blank an established tier", func(t *testing.T) { + entry := &logstore.Log{} + applyServedTierToEntry(entry, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ServiceTier: &flex}, + }, nil) + applyServedTierToEntry(entry, nil, &schemas.BifrostLLMUsage{}) + if entry.ServiceTier == nil || *entry.ServiceTier != "flex" { + t.Fatalf("expected service_tier to survive a tier-less update, got %v", entry.ServiceTier) + } + }) +} + +// TestCalculateCostForLogPricesOneHourCacheWrites covers the Responses-path leak: +// CachedWriteTokenDetails was dropped in the conversion, so 1h cache writes billed at +// the cheaper 5m rate. +func TestCalculateCostForLogPricesOneHourCacheWrites(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + const promptTokens, cachedWriteTokens, completionTokens = 20_000, 20_000, 100 + details := &schemas.ChatPromptTokensDetails{ + CachedWriteTokens: cachedWriteTokens, + CachedWriteTokenDetails: &schemas.ChatCachedWriteTokenDetails{ + CachedWriteTokens1h: cachedWriteTokens, + }, + } + usage := &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + PromptTokensDetails: details, + } + + want := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.Anthropic, Model: "claude-sonnet-4-20250514"}, + }, + }, + }, string(schemas.Anthropic)) + + fiveMinuteOnly := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{CachedWriteTokens: cachedWriteTokens}, + }, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.Anthropic, Model: "claude-sonnet-4-20250514"}, + }, + }, + }, string(schemas.Anthropic)) + if math.Abs(want-fiveMinuteOnly) <= costEpsilon { + t.Fatal("test datasheet must price 1h cache writes differently from 5m for this test to mean anything") + } + + entry := &logstore.Log{ + ID: "req-1h-cache", + Timestamp: time.Now().UTC(), + Object: string(schemas.ResponsesRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + TokenUsageParsed: usage, + } + + got, err := plugin.calculateCostForLog(entry) + if err != nil { + t.Fatalf("calculateCostForLog() error = %v", err) + } + assertCostsEqual(t, "1h cache-write responses request", got, want) +} + +// TestCalculateCostForLogUsesServerSideFallbackModel covers the routing leak. +// resolvePricing ranks ServerSideFallbackModel ahead of every other candidate because +// the tokens belong to the model that actually ran, but recomputation never put it on +// RoutingInfo — so a fallback-served turn was priced at the requested model's rates. +func TestCalculateCostForLogUsesServerSideFallbackModel(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + const promptTokens, completionTokens = 1_000, 200 + usage := &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + } + + // The row was requested as gpt-4o but Anthropic-style server-side fallback served + // a different model; pricing must follow the serving model. + served := "gpt-4o-mini" + want := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{ + Provider: schemas.OpenAI, + Model: "gpt-4o", + ServerSideFallbackModel: &served, + }, + }, + }, + }, string(schemas.OpenAI)) + + requestedModelCost := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.OpenAI, Model: "gpt-4o"}, + }, + }, + }, string(schemas.OpenAI)) + if math.Abs(want-requestedModelCost) <= costEpsilon { + t.Fatal("gpt-4o and gpt-4o-mini must price differently for this test to mean anything") + } + + entry := &logstore.Log{ + ID: "req-ssf", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.OpenAI), + Model: "gpt-4o", + Status: "success", + ServerSideFallbackModel: &served, + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + TokenUsageParsed: usage, + } + + got, err := plugin.calculateCostForLog(entry) + if err != nil { + t.Fatalf("calculateCostForLog() error = %v", err) + } + assertCostsEqual(t, "server-side-fallback-served request", got, want) +} + +// TestRecalculateCostsCountsUnpriceableSeparately checks the job-level accounting: a +// row whose pricing inputs cannot be recovered must be left untouched and reported as +// unpriceable, not merged into the ordinary skip count. Writing a wrong number is the +// failure mode this whole change exists to prevent, so the count has to be visible. +func TestRecalculateCostsCountsUnpriceableSeparately(t *testing.T) { + plugin := newCostFidelityPlugin(t) + store := plugin.store + ctx := context.Background() + + now := time.Now().UTC() + + // Priceable row. + if err := store.Create(ctx, testRecalculateCostLog("req-priceable", now, 100, 50)); err != nil { + t.Fatalf("Create() error = %v", err) + } + // Row that looks offloaded: denormalized counters but no token_usage payload. + if err := store.Create(ctx, &logstore.Log{ + ID: "req-unpriceable", + Timestamp: now.Add(time.Second), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + }); err != nil { + t.Fatalf("Create() error = %v", err) + } + + result, err := plugin.RecalculateCosts(ctx, logstore.SearchFilters{}, 10) + if err != nil { + t.Fatalf("RecalculateCosts() error = %v", err) + } + if result.Updated != 1 { + t.Fatalf("expected exactly the priceable row to update, got %+v", result) + } + if result.Unpriceable != 1 { + t.Fatalf("expected 1 unpriceable row, got %+v", result) + } + + unpriceable, err := store.FindByID(ctx, "req-unpriceable") + if err != nil { + t.Fatalf("FindByID() error = %v", err) + } + if unpriceable.Cost != nil { + t.Fatalf("expected an unpriceable row to keep a nil cost rather than a wrong one, got %v", *unpriceable.Cost) + } +} + +// legacyOffloadedStore emulates rows written while token_usage was offloaded to object +// storage: the DB row carries only the denormalized counters, and the real payload comes +// back through HydrateBillingChunk. This is the shape that produced the ~3x overbill. +// +// plugins/logging cannot build a real HybridLogStore — newHybridLogStore and the +// in-memory object store are unexported, and the only real backends are s3/gcs — so the +// store contract is emulated here and the genuine hydration is covered by +// TestHybrid_SearchLogsForBillingHydratesCacheBreakdown in the logstore package. +type legacyOffloadedStore struct { + logstore.LogStore + rows []logstore.Log + payloads map[string]string // id -> the real token_usage JSON, as if held in S3 + costs map[string]float64 + fetches int + // missing marks IDs whose object cannot be fetched, so the row stays degraded. + missing map[string]struct{} + backfilled map[string]logstore.BillingPayloadBackfill +} + +func (s *legacyOffloadedStore) SearchLogsForBilling(_ context.Context, _ logstore.SearchFilters, _ logstore.PaginationOptions) (*logstore.SearchResult, error) { + page := make([]logstore.Log, len(s.rows)) + copy(page, s.rows) + for i := range page { + // AfterFind runs on a real store; here it is explicit. With token_usage blank + // this builds the lossy stub and marks the row degraded. + if err := page[i].DeserializeFields(); err != nil { + return nil, err + } + } + return &logstore.SearchResult{ + Logs: page, + Stats: logstore.SearchStats{TotalRequests: int64(len(page))}, + }, nil +} + +// SearchLogs backs the count-only "how many still have no cost" query the recalc runs +// after its loop. It deliberately uses the cheap list projection there, so this needs +// to serve Stats.TotalRequests and nothing more. +func (s *legacyOffloadedStore) SearchLogs(_ context.Context, f logstore.SearchFilters, _ logstore.PaginationOptions) (*logstore.SearchResult, error) { + var remaining int64 + for _, r := range s.rows { + if f.MissingCostOnly && s.costs[r.ID] > 0 { + continue + } + remaining++ + } + return &logstore.SearchResult{Stats: logstore.SearchStats{TotalRequests: remaining}}, nil +} + +func (s *legacyOffloadedStore) HydrateBillingChunk(_ context.Context, logs []*logstore.Log) (logstore.BillingHydrationResult, error) { + var result logstore.BillingHydrationResult + for _, l := range logs { + // Mirrors the real gate: a row that already carries its usage needs no fetch. + if l == nil || !l.HasObject || l.TokenUsage != "" { + continue + } + if _, gone := s.missing[l.ID]; gone { + result.Unpriceable = append(result.Unpriceable, l.ID) + continue + } + s.fetches++ + l.TokenUsage = s.payloads[l.ID] + if err := l.DeserializeFields(); err != nil { + return result, err + } + result.Hydrated = append(result.Hydrated, l.ID) + } + return result, nil +} + +func (s *legacyOffloadedStore) BulkUpdateCost(_ context.Context, updates map[string]float64) error { + for id, c := range updates { + s.costs[id] = c + } + return nil +} + +// BulkBackfillBillingPayloads records the write-back so a test can assert that a +// recovered payload lands in the row, making the next recompute fetch-free. +func (s *legacyOffloadedStore) BulkBackfillBillingPayloads(_ context.Context, updates map[string]logstore.BillingPayloadBackfill) error { + if s.backfilled == nil { + s.backfilled = map[string]logstore.BillingPayloadBackfill{} + } + for id, b := range updates { + s.backfilled[id] = b + // Mirror what the DB would now hold, so a second pass sees the backfilled row. + for i := range s.rows { + if s.rows[i].ID == id { + s.rows[i].TokenUsage = b.TokenUsage + s.rows[i].CacheDebug = b.CacheDebug + } + } + } + return nil +} + +func TestPriceLogsInChunksBackfillsContentHiddenPricingData(t *testing.T) { + plugin := newCostFidelityPlugin(t) + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 100, + CompletionTokens: 10, + TotalTokens: 110, + } + serialized := &logstore.Log{TokenUsageParsed: usage} + require.NoError(t, serialized.SerializeFields()) + + batch := []logstore.Log{{ + ID: "hidden-retained", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.OpenAI), + Model: "gpt-4o", + Status: "success", + HasObject: true, + ContentHidden: true, + PromptTokens: usage.PromptTokens, + CompletionTokens: usage.CompletionTokens, + TotalTokens: usage.TotalTokens, + }} + require.NoError(t, batch[0].DeserializeFields()) + + store := &legacyOffloadedStore{ + payloads: map[string]string{"hidden-retained": serialized.TokenUsage}, + costs: map[string]float64{}, + missing: map[string]struct{}{}, + } + plugin.store = store + + outcomes, err := plugin.priceLogsInChunks(context.Background(), batch) + require.NoError(t, err) + require.Len(t, outcomes, 1) + require.NoError(t, outcomes[0].err) + require.Positive(t, outcomes[0].cost) + require.Contains(t, store.backfilled, "hidden-retained") + require.Equal(t, serialized.TokenUsage, store.backfilled["hidden-retained"].TokenUsage, + "pricing metadata belongs in log_store even when content is hidden") + require.Empty(t, batch[0].TokenUsage, "hydrated payload must be released after pricing") +} + +// TestRecalculateCostsOnLegacyOffloadedRowMatchesLiveCost is the end-to-end regression +// test for the reported bug: a cache-heavy row whose token_usage lives only in object +// storage must reprice to exactly what it was billed live, not to the ~3x that pricing +// the denormalized stub produces. +func TestRecalculateCostsOnLegacyOffloadedRowMatchesLiveCost(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + const ( + promptTokens = 100_000 + cachedReadTokens = 70_000 + cachedWriteTokens = 10_000 + completionTokens = 500 + ) + usage := &schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: cachedReadTokens, + CachedWriteTokens: cachedWriteTokens, + }, + } + + want := liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: usage, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.Anthropic, Model: "claude-sonnet-4-20250514"}, + }, + }, + }, string(schemas.Anthropic)) + require.Greater(t, want, 0.0, "live cost must be positive for the comparison to mean anything") + + // Serialize once to get the payload S3 would hold and the counters the DB keeps. + serialized := &logstore.Log{TokenUsageParsed: usage} + require.NoError(t, serialized.SerializeFields()) + + row := logstore.Log{ + ID: "legacy-offloaded", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + TokenUsage: "", // blanked at write time; the payload is in object storage + PromptTokens: serialized.PromptTokens, + CompletionTokens: serialized.CompletionTokens, + TotalTokens: serialized.TotalTokens, + CachedReadTokens: serialized.CachedReadTokens, + } + + store := &legacyOffloadedStore{ + rows: []logstore.Log{row}, + payloads: map[string]string{"legacy-offloaded": serialized.TokenUsage}, + costs: map[string]float64{}, + missing: map[string]struct{}{}, + } + plugin.store = store + + result, err := plugin.RecalculateCosts(context.Background(), logstore.SearchFilters{}, 10) + require.NoError(t, err) + require.Equal(t, 1, result.Updated, "the row must be repriced, not skipped: %+v", result) + require.Zero(t, result.Unpriceable) + require.Equal(t, 1, store.fetches, "the offloaded payload must be fetched exactly once") + + assertCostsEqual(t, "legacy offloaded cache-heavy row", store.costs["legacy-offloaded"], want) + + priceUsage := func(u *schemas.BifrostLLMUsage) float64 { + return liveCost(t, plugin, &schemas.BifrostResponse{ + ChatResponse: &schemas.BifrostChatResponse{ + Usage: u, + ExtraFields: schemas.BifrostResponseExtraFields{ + RequestType: schemas.ChatCompletionRequest, + RoutingInfo: schemas.RoutingInfo{Provider: schemas.Anthropic, Model: "claude-sonnet-4-20250514"}, + }, + }, + }, string(schemas.Anthropic)) + } + + // Prove hydration is load-bearing: the stub the row arrives with prices to a + // different number, so skipping the fetch would mis-bill. + // + // Note the direction. Now that cached_read_tokens is in the projection the stub + // keeps the large cheap bucket and loses only the cache-write total, so it + // *under*-bills by the write-rate delta. That is a much smaller error than the + // reported one — which is the point of the assertion below it. + stub := row + require.NoError(t, stub.DeserializeFields()) + require.True(t, stub.IsUsageDegraded()) + stubCost := priceUsage(stub.TokenUsageParsed) + require.Greater(t, math.Abs(stubCost-want), costEpsilon, + "pricing the un-hydrated stub must differ from the true cost, or hydration would be pointless") + + // And reproduce the reported ~3x: that required NO cache detail at all, which is + // what the old projection produced by never selecting cached_read_tokens. Cache + // reads are the biggest bucket at a tenth of the input rate, so losing them is + // what turned a mis-bill into a multiple. + cacheBlind := priceUsage(&schemas.BifrostLLMUsage{ + PromptTokens: promptTokens, + CompletionTokens: completionTokens, + TotalTokens: promptTokens + completionTokens, + }) + require.Greater(t, cacheBlind, want*1.5, + "a fully cache-blind price should be a multiple of the true cost; that gap is the reported bug") +} + +// TestRecalculateCostsLeavesUnfetchableLegacyRowAlone covers the other half: when the +// object cannot be read, the row must keep its existing cost rather than be repriced +// from the stub. +func TestRecalculateCostsLeavesUnfetchableLegacyRowAlone(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + row := logstore.Log{ + ID: "legacy-lost", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + } + store := &legacyOffloadedStore{ + rows: []logstore.Log{row}, + payloads: map[string]string{}, + costs: map[string]float64{}, + missing: map[string]struct{}{"legacy-lost": {}}, + } + plugin.store = store + + result, err := plugin.RecalculateCosts(context.Background(), logstore.SearchFilters{}, 10) + require.NoError(t, err) + assert.Zero(t, result.Updated, "an unfetchable row must not be repriced from the stub") + assert.Equal(t, 1, result.Unpriceable) + assert.Empty(t, store.costs, "no cost may be written for a row we cannot price") +} + +// TestRecalculateCostsRecordsZeroForUnfetchableDirectCacheHit covers the second place a +// row can be rejected before pricing: the hydration blocklist in priceLogsInChunks, +// which short-circuits before calculateCostForLog ever runs. +// +// A direct cache hit stays provably free even when its object is gone, because hit_type +// lives in the cache_debug DB column rather than the offloaded payload. Without the +// known-zero check in that branch the row is counted unpriceable and left with a nil +// cost, so every MissingCostOnly pass reattempts the same dead fetch forever. +func TestRecalculateCostsRecordsZeroForUnfetchableDirectCacheHit(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + row := logstore.Log{ + ID: "legacy-lost-hit", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + CachedReadTokens: 70_000, + CacheDebug: `{"cache_hit":true,"hit_type":"direct"}`, + } + store := &legacyOffloadedStore{ + rows: []logstore.Log{row}, + payloads: map[string]string{}, + costs: map[string]float64{}, + missing: map[string]struct{}{"legacy-lost-hit": {}}, + } + plugin.store = store + + result, err := plugin.RecalculateCosts(context.Background(), logstore.SearchFilters{}, 10) + require.NoError(t, err) + assert.Equal(t, 1, result.Updated, "a provably-free row must be closed out, not left unpriced") + assert.Zero(t, result.Unpriceable, "cost is knowable here, so the row is not unpriceable") + cost, ok := store.costs["legacy-lost-hit"] + require.True(t, ok, "the zero must actually be written so MissingCostOnly stops revisiting") + assert.Zero(t, cost) +} + +// TestRecalculateCostsBackfillsRecoveredUsageSoSecondRunSkipsObjectStore is the payoff +// of the write-back: recompute is self-healing. +// +// The first pass over a window pays one object fetch per offloaded row and writes the +// small pricing inputs into the row. Every pass after it reads them from the database, +// so the object store is not touched at all — and the cost comes out identical. +func TestRecalculateCostsBackfillsRecoveredUsageSoSecondRunSkipsObjectStore(t *testing.T) { + plugin := newCostFidelityPlugin(t) + + usage := &schemas.BifrostLLMUsage{ + PromptTokens: 100_000, + CompletionTokens: 500, + TotalTokens: 100_500, + PromptTokensDetails: &schemas.ChatPromptTokensDetails{ + CachedReadTokens: 70_000, + CachedWriteTokens: 10_000, + }, + } + serialized := &logstore.Log{TokenUsageParsed: usage} + require.NoError(t, serialized.SerializeFields()) + + store := &legacyOffloadedStore{ + rows: []logstore.Log{{ + ID: "backfill-1", + Timestamp: time.Now().UTC(), + Object: string(schemas.ChatCompletionRequest), + Provider: string(schemas.Anthropic), + Model: "claude-sonnet-4-20250514", + Status: "success", + HasObject: true, + PromptTokens: serialized.PromptTokens, + CompletionTokens: serialized.CompletionTokens, + TotalTokens: serialized.TotalTokens, + CachedReadTokens: serialized.CachedReadTokens, + }}, + payloads: map[string]string{"backfill-1": serialized.TokenUsage}, + costs: map[string]float64{}, + missing: map[string]struct{}{}, + } + plugin.store = store + + first, err := plugin.RecalculateCosts(context.Background(), logstore.SearchFilters{}, 10) + require.NoError(t, err) + require.Equal(t, 1, first.Updated) + require.Equal(t, 1, store.fetches, "the first pass must fetch the offloaded payload") + + // The recovered usage was written back to the row. + require.Contains(t, store.backfilled, "backfill-1", + "a fetched payload must be persisted, or every future run refetches it") + assert.Equal(t, serialized.TokenUsage, store.backfilled["backfill-1"].TokenUsage) + + firstCost := store.costs["backfill-1"] + require.Greater(t, firstCost, 0.0) + + // Second pass over the same window: same answer, no object store involvement. + store.costs = map[string]float64{} + second, err := plugin.RecalculateCosts(context.Background(), logstore.SearchFilters{}, 10) + require.NoError(t, err) + require.Equal(t, 1, second.Updated) + assert.Equal(t, 1, store.fetches, + "the second pass must be served entirely from the database; fetches should still be 1") + assertCostsEqual(t, "recompute after backfill", store.costs["backfill-1"], firstCost) +} diff --git a/plugins/logging/costrecalc.go b/plugins/logging/costrecalc.go index 1340479685e..06d34055f61 100644 --- a/plugins/logging/costrecalc.go +++ b/plugins/logging/costrecalc.go @@ -2,6 +2,7 @@ package logging import ( "context" + "errors" "fmt" "time" @@ -12,12 +13,13 @@ import ( // CostRecalcJobKind is the sidekiq job kind used for background cost recalculation. const CostRecalcJobKind = "logs_recalculate_cost" -// costRecalcBatchSize is the number of rows processed between checkpoints. Each -// batch is a single SearchLogs page followed by one BulkUpdateCost and one -// metadata checkpoint, so this also bounds how much work a crash can lose. It is -// a var (not a const) only so tests can shrink it to exercise multi-batch paths; -// production never reassigns it. -var costRecalcBatchSize = 1000 +// costRecalcBatchSize is both the billing-query page size and the number of rows +// processed between checkpoints. SearchLogsForBilling includes DB-resident modality +// payloads, so the page itself must be as small as the subsequent hydration chunk; +// otherwise a page of large image/audio payloads is already resident before chunked +// object-store hydration begins. It is a var only so tests can exercise multi-batch +// paths; production never reassigns it. +var costRecalcBatchSize = logstore.BillingHydrationChunkSize // CostRecalcJobMeta is the durable state of a cost-recalculation job. It is stored // verbatim as the sidekiq job's metadata JSON, so the worker can resume from the @@ -48,6 +50,13 @@ type CostRecalcJobMeta struct { Processed int `json:"processed"` Updated int `json:"updated"` Skipped int `json:"skipped"` + // Unpriceable is the subset of Skipped left alone because their pricing inputs + // could not be recovered — for example, an object-storage fetch that failed. + // Broken out because it means "we refused to write a number we knew + // would be wrong", which is actionable, whereas the rest of Skipped covers + // ordinary cases like a row with no usage to price. It is a subset rather than a + // sibling so Updated + Skipped still accounts for every Processed row. + Unpriceable int `json:"unpriceable,omitempty"` // Message carries a human-readable completion note for the UI. Message string `json:"message,omitempty"` } @@ -153,7 +162,11 @@ func (p *LoggerPlugin) RunCostRecalcJob(ctx context.Context, metaJSON string, ch filters.EndTime = windowEnd pagination.Offset = meta.CursorOffset - searchResult, err := p.store.SearchLogs(ctx, filters, pagination) + // Billing reads go through SearchLogsForBilling, not SearchLogs: the list + // projection omits the modality output payloads and, on object-storage-backed + // stores, returns rows whose token_usage was blanked at write time. Pricing + // those degraded rows charges cached tokens at the full input rate. + searchResult, err := p.store.SearchLogsForBilling(ctx, filters, pagination) if err != nil { return snapshot(), fmt.Errorf("failed to search logs for cost recalculation: %w", err) } @@ -162,19 +175,30 @@ func (p *LoggerPlugin) RunCostRecalcJob(ctx context.Context, metaJSON string, ch break } + // Hydrates and prices the payload-safe page, releasing its payloads before the + // next query. + outcomes, err := p.priceLogsInChunks(ctx, batch) + if err != nil { + return snapshot(), err + } + costUpdates := make(map[string]float64, len(batch)) gotPositiveCost := make([]bool, len(batch)) batchSkipped := 0 + batchUnpriceable := 0 for i := range batch { logEntry := batch[i] - cost, calcErr := p.calculateCostForLog(&logEntry) + cost, calcErr := outcomes[i].cost, outcomes[i].err if calcErr != nil { batchSkipped++ + if errors.Is(calcErr, errPricingInputsUnavailable) { + batchUnpriceable++ + } p.logger.Debug("skipping cost recalculation for log %s: %v", logEntry.ID, calcErr) continue } if cost <= 0 { - if isKnownZeroCostLog(&logEntry) { + if outcomes[i].knownZeroCost { costUpdates[logEntry.ID] = cost } else { batchSkipped++ @@ -192,9 +216,10 @@ func (p *LoggerPlugin) RunCostRecalcJob(ctx context.Context, metaJSON string, ch } meta.Updated += len(costUpdates) } - // Merge the skip count only once the batch is durably committed, so a retry + // Merge the skip counts only once the batch is durably committed, so a retry // after a BulkUpdateCost failure cannot double-count the same skipped rows. meta.Skipped += batchSkipped + meta.Unpriceable += batchUnpriceable meta.Processed += len(batch) // Advance the cursor. The lower bound is inclusive and rows that keep matching @@ -232,5 +257,8 @@ func (p *LoggerPlugin) RunCostRecalcJob(ctx context.Context, metaJSON string, ch } meta.Message = fmt.Sprintf("Recalculated %d cost value(s); %d skipped.", meta.Updated, meta.Skipped) + if meta.Unpriceable > 0 { + meta.Message += fmt.Sprintf(" %d of those were left unchanged because their pricing inputs were unavailable.", meta.Unpriceable) + } return snapshot(), nil } diff --git a/plugins/logging/costrecalc_test.go b/plugins/logging/costrecalc_test.go index ea66417ad5d..f02cb33a704 100644 --- a/plugins/logging/costrecalc_test.go +++ b/plugins/logging/costrecalc_test.go @@ -19,13 +19,15 @@ import ( // overridden. Any other call panics, surfacing an unexpected dependency. type fakeRecalcStore struct { logstore.LogStore - logs []logstore.Log // pre-sorted by (timestamp, id) - cost map[string]float64 - hasCost map[string]bool - updateCount map[string]int // successful BulkUpdateCost touches per id - searchCalls int - bulkCalls int - failBulkOnCall int // 1-based bulk call number to fail; 0 = never + logs []logstore.Log // pre-sorted by (timestamp, id) + cost map[string]float64 + hasCost map[string]bool + updateCount map[string]int // successful BulkUpdateCost touches per id + searchCalls int + bulkCalls int + failBulkOnCall int // 1-based bulk call number to fail; 0 = never + hydrateChunkSizes []int // sizes handed to HydrateBillingChunk, in order + backfilled []int // sizes handed to BulkBackfillBillingPayloads, in order } func newFakeRecalcStore(logs []logstore.Log) *fakeRecalcStore { @@ -37,6 +39,28 @@ func newFakeRecalcStore(logs []logstore.Log) *fakeRecalcStore { } } +// SearchLogsForBilling is what the recalc job actually calls. The fake has nothing +// offloaded to object storage, so it returns the same rows as SearchLogs — matching +// RDBLogStore, where every pricing input is already present in the row. +func (s *fakeRecalcStore) SearchLogsForBilling(ctx context.Context, f logstore.SearchFilters, p logstore.PaginationOptions) (*logstore.SearchResult, error) { + return s.SearchLogs(ctx, f, p) +} + +// HydrateBillingChunk records the chunk sizes it is handed so tests can assert the +// caller never asks for more payloads at once than BillingHydrationChunkSize. Nothing +// is offloaded here, so there is nothing to fetch. +func (s *fakeRecalcStore) HydrateBillingChunk(_ context.Context, logs []*logstore.Log) (logstore.BillingHydrationResult, error) { + s.hydrateChunkSizes = append(s.hydrateChunkSizes, len(logs)) + return logstore.BillingHydrationResult{}, nil +} + +// BulkBackfillBillingPayloads should never be reached here: nothing is offloaded, so no +// row is ever hydrated and there is nothing recovered to write back. +func (s *fakeRecalcStore) BulkBackfillBillingPayloads(_ context.Context, updates map[string]logstore.BillingPayloadBackfill) error { + s.backfilled = append(s.backfilled, len(updates)) + return nil +} + func (s *fakeRecalcStore) SearchLogs(_ context.Context, f logstore.SearchFilters, p logstore.PaginationOptions) (*logstore.SearchResult, error) { s.searchCalls++ if s.searchCalls > 1000 { @@ -375,3 +399,45 @@ func TestRunCostRecalcJob_EmptyWindow(t *testing.T) { t.Errorf("expected no checkpoints for an empty window, got %d", len(checkpoints)) } } + +// TestRunCostRecalcJob_HydratesInBoundedChunks pins the memory bound. +// +// A hydrated row holds its whole offloaded payload — potentially full message +// histories and raw request/response bodies — so a recompute must never ask for a +// whole batch of them at once. The job walks the batch in BillingHydrationChunkSize +// slices and releases each before advancing, which keeps peak payload memory flat no +// matter how large the batch or the recompute window is. +func TestRunCostRecalcJob_HydratesInBoundedChunks(t *testing.T) { + base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + const rows = 10 + logs := make([]logstore.Log, 0, rows) + for i := range rows { + logs = append(logs, positiveLog(fmt.Sprintf("chunk-%02d", i), base.Add(time.Duration(i)*time.Second))) + } + store := newFakeRecalcStore(logs) + p := newRecalcPlugin(t, store) + + final, _, err := runJob(t, p, CostRecalcJobMeta{Filters: window(base)}) + if err != nil { + t.Fatalf("RunCostRecalcJob() error = %v", err) + } + if final.Updated != rows { + t.Fatalf("expected all %d rows priced, got Updated=%d", rows, final.Updated) + } + + if len(store.hydrateChunkSizes) == 0 { + t.Fatal("expected the job to hydrate through HydrateBillingChunk, but it never called it") + } + total := 0 + for _, size := range store.hydrateChunkSizes { + if size > logstore.BillingHydrationChunkSize { + t.Fatalf("hydration chunk of %d exceeds BillingHydrationChunkSize=%d; peak payload memory would scale with batch size, got chunks %v", + size, logstore.BillingHydrationChunkSize, store.hydrateChunkSizes) + } + total += size + } + if total != rows { + t.Fatalf("chunks covered %d rows, want every one of %d: %v", total, rows, store.hydrateChunkSizes) + } +} diff --git a/plugins/logging/main.go b/plugins/logging/main.go index 9c6b9422ad8..d59c396ba99 100644 --- a/plugins/logging/main.go +++ b/plugins/logging/main.go @@ -376,7 +376,13 @@ type RecalculateCostResult struct { TotalMatched int64 `json:"total_matched"` Updated int `json:"updated"` Skipped int `json:"skipped"` - Remaining int64 `json:"remaining"` + // Unpriceable is the subset of Skipped left alone because their pricing inputs + // could not be recovered (for example, a failed object-storage fetch) rather than + // because there was nothing to charge. Surfaced separately so + // a caller can distinguish "no cost applies" from "we refused to write a number + // we knew would be wrong". + Unpriceable int `json:"unpriceable,omitempty"` + Remaining int64 `json:"remaining"` } // RecalculateCostProgress represents a progress event from a cost backfill operation. diff --git a/plugins/logging/operations.go b/plugins/logging/operations.go index 8856bd1bb0e..4fb9129fbdd 100644 --- a/plugins/logging/operations.go +++ b/plugins/logging/operations.go @@ -3,6 +3,7 @@ package logging import ( "context" + "errors" "fmt" "strings" "time" @@ -406,6 +407,13 @@ func (p *LoggerPlugin) applyStreamingOutputToEntry(entry *logstore.Log, streamRe entry.CompletionTokens = streamResponse.Data.TokenUsage.CompletionTokens entry.TotalTokens = streamResponse.Data.TokenUsage.TotalTokens } + if streamResponse.Data.ServiceTier != nil { + entry.ServiceTier = new(string(*streamResponse.Data.ServiceTier)) + } + // Speed/InferenceGeo come off the usage struct (the provider sets them there for + // exactly this reason). ServiceTier is accumulated separately from the streamed + // response envelope above. + applyServedTierToEntry(entry, nil, streamResponse.Data.TokenUsage) // Cost if streamResponse.Data.Cost != nil { @@ -475,6 +483,66 @@ func (p *LoggerPlugin) applyStreamingOutputToEntry(entry *logstore.Log, streamRe } } +// applyServedTierToEntry records the billing tier the provider actually served — +// OpenAI's service_tier (priority/flex) and Anthropic's speed (fast mode) and +// inference_geo (data residency). All three scale token rates, so cost +// recomputation cannot reprice a row without them. +// +// These need dedicated columns rather than riding along in token_usage: +// BifrostLLMUsage tags Speed and InferenceGeo `json:"-"`, and service_tier lives on +// the response rather than on usage at all. The columns are also outside the +// payload set, so they survive hybrid offload and content-hidden rows. +// +// The values are the *served* tier, not the requested one. Providers echo what they +// actually did, so a request that asked for fast mode but fell back reports +// "standard" and must bill at standard rates. +// usage may be nil; it is the fallback source for Speed and InferenceGeo on the +// streaming path, where the accumulator rebuilds a response envelope that carries +// neither but the provider does populate them on the usage struct. +func applyServedTierToEntry(entry *logstore.Log, result *schemas.BifrostResponse, usage *schemas.BifrostLLMUsage) { + if entry == nil { + return + } + + var ( + serviceTier *schemas.BifrostServiceTier + speed *string + inferenceGeo *string + ) + if result != nil { + switch { + case result.ChatResponse != nil: + serviceTier, speed, inferenceGeo = result.ChatResponse.ServiceTier, result.ChatResponse.Speed, result.ChatResponse.InferenceGeo + case result.ResponsesResponse != nil: + serviceTier, speed, inferenceGeo = result.ResponsesResponse.ServiceTier, result.ResponsesResponse.Speed, result.ResponsesResponse.InferenceGeo + case result.ResponsesStreamResponse != nil && result.ResponsesStreamResponse.Response != nil: + r := result.ResponsesStreamResponse.Response + serviceTier, speed, inferenceGeo = r.ServiceTier, r.Speed, r.InferenceGeo + } + } + if usage != nil { + if speed == nil { + speed = usage.Speed + } + if inferenceGeo == nil { + inferenceGeo = usage.InferenceGeo + } + } + + // Only overwrite on a non-nil value. Streaming assembles an entry across many + // chunks and the tier arrives on whichever chunk carries the response envelope; + // a later usage-only chunk must not blank what an earlier one established. + if serviceTier != nil { + entry.ServiceTier = new(string(*serviceTier)) + } + if speed != nil { + entry.Speed = speed + } + if inferenceGeo != nil { + entry.InferenceGeo = inferenceGeo + } +} + // isPassthroughErrorResponse returns true when the result is a passthrough // response with a provider-reported HTTP error status (4xx or 5xx). func isPassthroughErrorResponse(result *schemas.BifrostResponse) bool { @@ -535,6 +603,7 @@ func (p *LoggerPlugin) applyNonStreamingOutputToEntry(entry *logstore.Log, resul entry.CompletionTokens = usage.CompletionTokens entry.TotalTokens = usage.TotalTokens } + applyServedTierToEntry(entry, result, usage) // Extract raw request/response and output content extraFields := result.GetExtraFields() @@ -645,6 +714,7 @@ func (p *LoggerPlugin) applyRealtimeOutputToEntry(entry *logstore.Log, result *s entry.PromptTokens = bifrostUsage.PromptTokens entry.CompletionTokens = bifrostUsage.CompletionTokens entry.TotalTokens = bifrostUsage.TotalTokens + applyServedTierToEntry(entry, result, bifrostUsage) } if contentLoggingEnabled { @@ -1270,8 +1340,10 @@ func (p *LoggerPlugin) RecalculateCostsWithProgress(ctx context.Context, filters if limit <= 0 { limit = 200 } - if limit > 1000 { - limit = 1000 + // SearchLogsForBilling materializes DB-resident modality outputs, so cap the + // query page itself to the same payload-safe size as the background job. + if limit > costRecalcBatchSize { + limit = costRecalcBatchSize } // filters.MissingCostOnly controls the scope: @@ -1293,7 +1365,10 @@ func (p *LoggerPlugin) RecalculateCostsWithProgress(ctx context.Context, filters for { pagination.Offset = remainingOffset - searchResult, err := p.store.SearchLogs(ctx, filters, pagination) + // Billing projection, not the list projection: see SearchLogsForBilling. The + // list rows omit the modality output payloads and, where payloads are + // offloaded, carry a usage stub that prices cached tokens at full rate. + searchResult, err := p.store.SearchLogsForBilling(ctx, filters, pagination) if err != nil { return nil, fmt.Errorf("failed to search logs for cost recalculation: %w", err) } @@ -1306,19 +1381,28 @@ func (p *LoggerPlugin) RecalculateCostsWithProgress(ctx context.Context, filters } processed += len(searchResult.Logs) + outcomes, err := p.priceLogsInChunks(ctx, searchResult.Logs) + if err != nil { + return nil, err + } + costUpdates := make(map[string]float64, len(searchResult.Logs)) stillMissingInBatch := 0 - for _, logEntry := range searchResult.Logs { - cost, calcErr := p.calculateCostForLog(&logEntry) + for i := range searchResult.Logs { + logEntry := searchResult.Logs[i] + cost, calcErr := outcomes[i].cost, outcomes[i].err if calcErr != nil { result.Skipped++ + if errors.Is(calcErr, errPricingInputsUnavailable) { + result.Unpriceable++ + } stillMissingInBatch++ p.logger.Debug("skipping cost recalculation for log %s: %v", logEntry.ID, calcErr) continue } if cost <= 0 { - if isKnownZeroCostLog(&logEntry) { + if outcomes[i].knownZeroCost { costUpdates[logEntry.ID] = cost } else { result.Skipped++ @@ -1390,6 +1474,109 @@ func (p *LoggerPlugin) RecalculateCostsWithProgress(ctx context.Context, filters return result, nil } +// errPricingInputsUnavailable marks a log whose pricing inputs could not be +// recovered, as distinct from one that merely priced to zero. The recalc job +// reports the two separately so operators can tell "nothing to charge" from +// "could not compute a charge". +var errPricingInputsUnavailable = errors.New("pricing inputs unavailable") + +// billingOutcome is the per-row result of pricing a batch: the computed cost, or the +// reason the row was left alone. +type billingOutcome struct { + cost float64 + err error + // knownZeroCost is captured while the row's payload is still hydrated, because it + // is derived from CacheDebugParsed and ReleaseBillingPayloads clears that. Callers + // run after the release, so they cannot re-derive it. + knownZeroCost bool +} + +// priceLogsInChunks hydrates and prices a batch a few rows at a time, releasing each +// chunk's payloads before moving to the next, and returns one outcome per input row. +// +// Billing query pages are capped at BillingHydrationChunkSize because DB-resident +// modality outputs are materialized by the query. Chunking at the same size bounds +// object-store hydration too: a hydrated row can include full message histories and +// raw request/response bodies. Each page is released before the next query, so peak +// payload memory does not scale with the recompute window. +// +// Rows the store could not hydrate are marked errPricingInputsUnavailable rather than +// priced, so an unrecoverable payload can never be billed from the lossy fallback. +func (p *LoggerPlugin) priceLogsInChunks(ctx context.Context, batch []logstore.Log) ([]billingOutcome, error) { + outcomes := make([]billingOutcome, len(batch)) + + // Pricing inputs recovered from object storage, written back once at the end so a + // later recompute reads them from the DB instead of fetching again. Accumulating + // across chunks rather than flushing per chunk keeps the write count down; it is + // safe for memory because only the two small fields are kept, not the payloads the + // release below drops. + backfill := map[string]logstore.BillingPayloadBackfill{} + + for start := 0; start < len(batch); start += logstore.BillingHydrationChunkSize { + end := min(start+logstore.BillingHydrationChunkSize, len(batch)) + + chunk := make([]*logstore.Log, 0, end-start) + for i := start; i < end; i++ { + chunk = append(chunk, &batch[i]) + } + + hydration, err := p.store.HydrateBillingChunk(ctx, chunk) + if err != nil { + return nil, fmt.Errorf("failed to hydrate pricing inputs: %w", err) + } + blocked := make(map[string]struct{}, len(hydration.Unpriceable)) + for _, id := range hydration.Unpriceable { + blocked[id] = struct{}{} + } + // Only rows the store actually fetched are worth writing back; everything else + // already came from the database. + fetched := make(map[string]struct{}, len(hydration.Hydrated)) + for _, id := range hydration.Hydrated { + fetched[id] = struct{}{} + } + + for i := start; i < end; i++ { + if _, isBlocked := blocked[batch[i].ID]; isBlocked { + // A direct cache hit costs nothing whether or not its payload came back — + // cache_debug is a DB column, so the hit type survives a failed or refused + // object fetch. Record the zero instead of leaving the row permanently + // unpriced for every MissingCostOnly pass to revisit. + if isKnownZeroCostLog(&batch[i]) { + outcomes[i].knownZeroCost = true + continue + } + outcomes[i].err = fmt.Errorf("%w: log %s", errPricingInputsUnavailable, batch[i].ID) + continue + } + outcomes[i].cost, outcomes[i].err = p.calculateCostForLog(&batch[i]) + // Must be read here, before the release below drops cache_debug. + outcomes[i].knownZeroCost = isKnownZeroCostLog(&batch[i]) + // Pricing metadata belongs in the log store even when request/response + // content is hidden. Only the small token_usage and cache_debug fields are + // backfilled; content-bearing payload fields remain in object storage. + if _, wasFetched := fetched[batch[i].ID]; wasFetched { + backfill[batch[i].ID] = logstore.BillingPayloadBackfill{ + TokenUsage: batch[i].TokenUsage, + CacheDebug: batch[i].CacheDebug, + } + } + } + + // Release before advancing so at most one chunk of payloads is ever resident. + logstore.ReleaseBillingPayloads(chunk) + } + + if len(backfill) > 0 { + // Non-fatal: the cost update is this job's real output, and a missed backfill + // only means the next run pays the fetches again. + if err := p.store.BulkBackfillBillingPayloads(ctx, backfill); err != nil { + p.logger.Warn("failed to backfill recovered pricing inputs for %d log(s); future recalculations will refetch them: %v", len(backfill), err) + } + } + + return outcomes, nil +} + func isKnownZeroCostLog(logEntry *logstore.Log) bool { if logEntry == nil || logEntry.CacheDebugParsed == nil || !logEntry.CacheDebugParsed.CacheHit { return false @@ -1430,6 +1617,26 @@ func (p *LoggerPlugin) calculateCostForLog(logEntry *logstore.Log) (float64, err return 0, fmt.Errorf("token usage not available for log %s", logEntry.ID) } + // A direct cache hit was served without an LLM call, so pricing returns zero + // regardless of the token breakdown (see calculateCostWithCache). Short-circuiting + // ahead of the degraded gate keeps a provably-free row from being reported + // unpriceable and revisited by every MissingCostOnly pass. + if isKnownZeroCostLog(logEntry) { + return 0, nil + } + + // Refuse to price a usage stub rebuilt from denormalized columns. It lacks the + // cache-write/1h split, the audio and search-query details, and cache_debug — + // and because PromptTokens is inclusive of the cache buckets, pricing it charges + // every cached token at the full input rate and can inflate a cache-heavy + // request several fold. SearchLogsForBilling hydrates whatever it can, so a row + // still degraded here is genuinely unpriceable (for example, the object fetch + // failed). Erroring makes the recalc job count it as skipped instead of + // writing a number that is wrong by multiples. + if logEntry.IsUsageDegraded() { + return 0, fmt.Errorf("%w: log %s", errPricingInputsUnavailable, logEntry.ID) + } + requestType := normalizeLogRequestType(logEntry.Object) if requestType == "" && (cacheDebug == nil || !cacheDebug.CacheHit) { p.logger.Warn("skipping cost calculation for log %s: object type is empty (timestamp: %s)", logEntry.ID, logEntry.Timestamp) @@ -1452,6 +1659,11 @@ func (p *LoggerPlugin) calculateCostForLog(logEntry *logstore.Log) (float64, err RoutingInfo: schemas.RoutingInfo{ Provider: schemas.ModelProvider(logEntry.Provider), Model: originalModelRequested, + // resolvePricing ranks ServerSideFallbackModel ahead of every other + // candidate because the tokens being priced belong to the model that + // actually ran, not the one the caller asked for. Anthropic's + // server-side fallback is the only producer today. + ServerSideFallbackModel: logEntry.ServerSideFallbackModel, }, } @@ -1459,15 +1671,24 @@ func (p *LoggerPlugin) calculateCostForLog(logEntry *logstore.Log) (float64, err // pricing the same routing info live logging had. Without this the canonical // model name is dropped and pricing only tries the wire model / alias name, // nulling costs that live logging resolved via the canonical name. + // + // ModelID is populated whenever an alias matched, independent of whether a + // canonical name was configured on it. resolvePricing derives its override key + // from ModelID, so leaving it empty on a canonical-less alias would look up + // per-deployment override pricing under the alias name instead of the wire + // model and silently miss it. + if logEntry.Alias != nil && *logEntry.Alias != "" { + extraFields.RoutingInfo.ResolvedKeyAlias = &schemas.ResolvedKeyAlias{ModelID: logEntry.Model} + } if logEntry.CanonicalModelName != nil && *logEntry.CanonicalModelName != "" { canonical := *logEntry.CanonicalModelName - extraFields.RoutingInfo.ResolvedKeyAlias = &schemas.ResolvedKeyAlias{ - ModelID: logEntry.Model, - ModelName: &canonical, + if extraFields.RoutingInfo.ResolvedKeyAlias == nil { + extraFields.RoutingInfo.ResolvedKeyAlias = &schemas.ResolvedKeyAlias{ModelID: logEntry.Model} } + extraFields.RoutingInfo.ResolvedKeyAlias.ModelName = &canonical } - resp := buildResponseForRequestType(requestType, usage, extraFields) + resp := buildResponseForRequestType(requestType, usage, extraFields, servedTierFromLog(logEntry)) // Patch modality-specific output fields that are not captured in BifrostLLMUsage // but are required for accurate cost calculation. @@ -1502,6 +1723,15 @@ func (p *LoggerPlugin) calculateCostForLog(logEntry *logstore.Log) (float64, err resp.VideoGenerationResponse.Seconds = logEntry.VideoGenerationOutputParsed.Seconds } + // OCR: restore Pages and UsageInfo. OCR bills per page processed and nothing + // else, so without this the reconstructed response reports zero pages and the + // whole request prices to nothing. + if resp.OCRResponse != nil && logEntry.OCROutputParsed != nil { + resp.OCRResponse.Pages = logEntry.OCROutputParsed.Pages + resp.OCRResponse.UsageInfo = logEntry.OCROutputParsed.UsageInfo + resp.OCRResponse.DocumentAnnotation = logEntry.OCROutputParsed.DocumentAnnotation + } + // Speech: restore provider-specific usage (e.g. character-count billing) from // the stored response instead of relying solely on aggregate token counts. if resp.SpeechResponse != nil && @@ -1514,9 +1744,36 @@ func (p *LoggerPlugin) calculateCostForLog(logEntry *logstore.Log) (float64, err return p.pricingManager.CalculateCost(resp, &scopes), nil } +// servedTier carries the billing tier a log row was served at, read back from the +// denormalized columns. CalculateCost derives its rate multipliers from these via +// tierFromResponse, so they have to be put back onto the reconstructed response or +// every row reprices at standard rates. +type servedTier struct { + serviceTier *schemas.BifrostServiceTier + speed *string + inferenceGeo *string +} + +// servedTierFromLog reads the served tier off the log's dedicated columns. These +// cannot come from token_usage: BifrostLLMUsage tags Speed and InferenceGeo +// `json:"-"`, and service_tier lives on the response rather than on usage at all. +// Rows written before those columns existed return an empty tier and reprice at +// standard rates — the honest outcome, since the information was never captured. +func servedTierFromLog(logEntry *logstore.Log) servedTier { + if logEntry == nil { + return servedTier{} + } + tier := servedTier{speed: logEntry.Speed, inferenceGeo: logEntry.InferenceGeo} + if logEntry.ServiceTier != nil && *logEntry.ServiceTier != "" { + st := schemas.BifrostServiceTier(*logEntry.ServiceTier) + tier.serviceTier = &st + } + return tier +} + // buildResponseForRequestType wraps BifrostLLMUsage into the correct response // field so that CalculateCost's extractCostInput routes it properly. -func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas.BifrostLLMUsage, extra schemas.BifrostResponseExtraFields) *schemas.BifrostResponse { +func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas.BifrostLLMUsage, extra schemas.BifrostResponseExtraFields, tier servedTier) *schemas.BifrostResponse { switch requestType { case schemas.TextCompletionRequest, schemas.TextCompletionStreamRequest: return &schemas.BifrostResponse{ @@ -1563,6 +1820,8 @@ func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas ImageTokens: usage.PromptTokensDetails.ImageTokens, CachedReadTokens: usage.PromptTokensDetails.CachedReadTokens, CachedWriteTokens: usage.PromptTokensDetails.CachedWriteTokens, + // The 5m/1h split drives tieredCacheCreationInputAbove1hrTokenRate. + // Dropping it silently bills 1h cache writes at the cheaper 5m rate. CachedWriteTokenDetails: usage.PromptTokensDetails.CachedWriteTokenDetails, } } @@ -1581,8 +1840,11 @@ func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas } return &schemas.BifrostResponse{ ResponsesResponse: &schemas.BifrostResponsesResponse{ - Usage: respUsage, - ExtraFields: extra, + Usage: respUsage, + ExtraFields: extra, + ServiceTier: tier.serviceTier, + Speed: tier.speed, + InferenceGeo: tier.inferenceGeo, }, } case schemas.SpeechRequest, schemas.SpeechStreamRequest: @@ -1644,8 +1906,11 @@ func buildResponseForRequestType(requestType schemas.RequestType, usage *schemas // Default to chat response for unknown or chat request types return &schemas.BifrostResponse{ ChatResponse: &schemas.BifrostChatResponse{ - Usage: usage, - ExtraFields: extra, + Usage: usage, + ExtraFields: extra, + ServiceTier: tier.serviceTier, + Speed: tier.speed, + InferenceGeo: tier.inferenceGeo, }, } } diff --git a/plugins/logging/operations_test.go b/plugins/logging/operations_test.go index 3f115e7840d..24a950eed21 100644 --- a/plugins/logging/operations_test.go +++ b/plugins/logging/operations_test.go @@ -440,6 +440,23 @@ func TestApplyStreamingOutputToEntryPreservesAccumulatorCancelledStatus(t *testi } } +func TestApplyStreamingOutputToEntryPreservesServiceTier(t *testing.T) { + plugin := &LoggerPlugin{} + entry := &logstore.Log{} + priority := schemas.BifrostServiceTierPriority + + plugin.applyStreamingOutputToEntry(entry, &streaming.ProcessedStreamResponse{ + Data: &streaming.AccumulatedData{ + ServiceTier: &priority, + TokenUsage: &schemas.BifrostLLMUsage{TotalTokens: 1}, + }, + }, false, true) + + if entry.ServiceTier == nil || *entry.ServiceTier != string(schemas.BifrostServiceTierPriority) { + t.Fatalf("expected priority service tier, got %v", entry.ServiceTier) + } +} + // newTestPricingManager builds a ModelCatalog backed by the committed pricing // testdata via an offline file:// URL (no network). func newTestPricingManager(t *testing.T) *modelcatalog.ModelCatalog { diff --git a/transports/bifrost-http/handlers/logging.go b/transports/bifrost-http/handlers/logging.go index 453fe00d2be..4edb1f76b24 100644 --- a/transports/bifrost-http/handlers/logging.go +++ b/transports/bifrost-http/handlers/logging.go @@ -1998,16 +1998,22 @@ func (h *LoggingHandler) getRecalculateCostStatus(ctx *fasthttp.RequestCtx) { // recalcJobStatus is the API view of a cost-recalculation job: the durable sidekiq // row fields the UI needs plus the progress counters decoded from the job metadata. type recalcJobStatus struct { - ID string `json:"id,omitempty"` - Status string `json:"status"` - Total int64 `json:"total"` - Processed int `json:"processed"` - Updated int `json:"updated"` - Skipped int `json:"skipped"` - Message string `json:"message,omitempty"` - LastError string `json:"last_error,omitempty"` - StartedAt *time.Time `json:"started_at,omitempty"` - UpdatedAt *time.Time `json:"updated_at,omitempty"` + ID string `json:"id,omitempty"` + Status string `json:"status"` + Total int64 `json:"total"` + Processed int `json:"processed"` + Updated int `json:"updated"` + Skipped int `json:"skipped"` + // Unpriceable is the subset of Skipped whose pricing inputs could not be + // recovered, so the job deliberately left their cost untouched rather than + // writing a value it knew to be wrong. Surfaced separately because it is + // actionable: it usually means an offloaded payload is missing from object + // storage, or the rows are content-hidden and can never be repriced. + Unpriceable int `json:"unpriceable,omitempty"` + Message string `json:"message,omitempty"` + LastError string `json:"last_error,omitempty"` + StartedAt *time.Time `json:"started_at,omitempty"` + UpdatedAt *time.Time `json:"updated_at,omitempty"` } // recalcJobStatusFromRow projects a sidekiq job row into the API status, decoding @@ -2028,6 +2034,7 @@ func recalcJobStatusFromRow(job *tables.TableSidekiqJob) recalcJobStatus { status.Processed = meta.Processed status.Updated = meta.Updated status.Skipped = meta.Skipped + status.Unpriceable = meta.Unpriceable status.Message = meta.Message } }