From 61f7e27ccd51cfea7653cf8d5d8c3803d7002e7e Mon Sep 17 00:00:00 2001 From: roroghost17 Date: Fri, 22 May 2026 12:18:08 +0530 Subject: [PATCH 1/3] fix: search content and video getting dropped from Bedrock requests --- core/providers/bedrock/bedrock_test.go | 285 +++++++++++++++++++++++++ core/providers/bedrock/responses.go | 26 ++- core/providers/bedrock/types.go | 49 +++++ core/providers/bedrock/utils.go | 40 ++++ 4 files changed, 397 insertions(+), 3 deletions(-) diff --git a/core/providers/bedrock/bedrock_test.go b/core/providers/bedrock/bedrock_test.go index 6be0f733640..b4f80ab3444 100644 --- a/core/providers/bedrock/bedrock_test.go +++ b/core/providers/bedrock/bedrock_test.go @@ -5233,3 +5233,288 @@ func TestBedrockToBifrostResponsesResponse_StructuredOutput_MixedWithRealTools(t assert.Equal(t, "tool_calls", *result.StopReason, "expected stop_reason=tool_calls when real tool calls are also present") } + +// TestBedrockSearchResultToolResultRoundTrip is the regression gate for +// https://github.com/maximhq/bifrost/issues/3537 — a Bedrock-native passthrough +// request containing toolResult.content[].searchResult must survive +// ToBifrostResponsesRequest → ToBedrockResponsesRequest with all fields intact. +// Pre-fix, the SearchResult field is dropped during JSON unmarshal and the +// outbound request shows toolResult.content = [{"text": ""}]. +func TestBedrockSearchResultToolResultRoundTrip(t *testing.T) { + original := &bedrock.BedrockConverseRequest{ + ModelID: "anthropic.claude-sonnet-4-5", + Messages: []bedrock.BedrockMessage{ + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + {Text: schemas.Ptr("What is Apptio?")}, + }, + }, + { + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "tooluse_a4rBqeZNRTKj2lTskvaO4H", + Name: "RAGRequest", + Input: json.RawMessage(`{"query":"What is Apptio?"}`), + }, + }, + }, + }, + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + { + ToolResult: &bedrock.BedrockToolResult{ + ToolUseID: "tooluse_a4rBqeZNRTKj2lTskvaO4H", + Status: schemas.Ptr("success"), + Content: []bedrock.BedrockContentBlock{ + { + SearchResult: &bedrock.BedrockSearchResultBlock{ + Source: "Great Source of Information About Apptio", + Title: "12adbd74-46bd-4a88-88b2-0048755f6eb5", + Content: []bedrock.BedrockSearchResultContent{ + {Text: "Apptio is a company that makes calls to Bedrock using passthrough APIs via Bifrost"}, + }, + Citations: &bedrock.BedrockCitationsConfig{Enabled: true}, + }, + }, + }, + }, + }, + }, + }, + }, + System: []bedrock.BedrockSystemMessage{ + {Text: schemas.Ptr("Do not rely on your knowledge to answer. Use only the tool results.")}, + }, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + // First leg: Bedrock → Bifrost intermediate. + bifrostReq, err := original.ToBifrostResponsesRequest(ctx) + require.NoError(t, err) + require.NotNil(t, bifrostReq) + + // Second leg: Bifrost intermediate → Bedrock. + rebuilt, err := bedrock.ToBedrockResponsesRequest(ctx, bifrostReq) + require.NoError(t, err) + require.NotNil(t, rebuilt) + + // Locate the rebuilt toolResult content block (its position may shift + // because the converter groups assistant tool calls and user tool results + // by state-machine emission, but a toolResult with our toolUseId must exist). + var got *bedrock.BedrockSearchResultBlock + for _, msg := range rebuilt.Messages { + for _, block := range msg.Content { + if block.ToolResult == nil { + continue + } + if block.ToolResult.ToolUseID != "tooluse_a4rBqeZNRTKj2lTskvaO4H" { + continue + } + for _, c := range block.ToolResult.Content { + if c.SearchResult != nil { + got = c.SearchResult + break + } + } + } + } + + require.NotNil(t, got, "expected toolResult.content[].searchResult to round-trip; got nil (regression of #3537)") + assert.Equal(t, "Great Source of Information About Apptio", got.Source) + assert.Equal(t, "12adbd74-46bd-4a88-88b2-0048755f6eb5", got.Title) + require.Len(t, got.Content, 1) + assert.Equal(t, "Apptio is a company that makes calls to Bedrock using passthrough APIs via Bifrost", got.Content[0].Text) + require.NotNil(t, got.Citations) + assert.True(t, got.Citations.Enabled) +} + +// TestBedrockVideoToolResultRoundTrip verifies that a video block inside +// toolResult.content survives ToBifrostResponsesRequest → ToBedrockResponsesRequest +// via the same sentinel-envelope mechanism that preserves searchResult. Without +// the schema fix + envelope trigger extension, Video is silently dropped at JSON +// unmarshal and the outbound request carries an empty text block instead. +func TestBedrockVideoToolResultRoundTrip(t *testing.T) { + // Smallest plausible base64 payload — content doesn't matter for the round-trip, + // only that the Video struct is preserved verbatim. + videoBytes := "AAAA" + + original := &bedrock.BedrockConverseRequest{ + ModelID: "anthropic.claude-sonnet-4-5", + Messages: []bedrock.BedrockMessage{ + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + {Text: schemas.Ptr("Describe the attached clip.")}, + }, + }, + { + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "tooluse_video_xyz", + Name: "FetchClip", + Input: json.RawMessage(`{"id":"abc"}`), + }, + }, + }, + }, + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + { + ToolResult: &bedrock.BedrockToolResult{ + ToolUseID: "tooluse_video_xyz", + Status: schemas.Ptr("success"), + Content: []bedrock.BedrockContentBlock{ + { + Video: &bedrock.BedrockVideoBlock{ + Format: "mp4", + Source: bedrock.BedrockVideoSource{ + Bytes: &videoBytes, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + bifrostReq, err := original.ToBifrostResponsesRequest(ctx) + require.NoError(t, err) + require.NotNil(t, bifrostReq) + + rebuilt, err := bedrock.ToBedrockResponsesRequest(ctx, bifrostReq) + require.NoError(t, err) + require.NotNil(t, rebuilt) + + var got *bedrock.BedrockVideoBlock + for _, msg := range rebuilt.Messages { + for _, block := range msg.Content { + if block.ToolResult == nil || block.ToolResult.ToolUseID != "tooluse_video_xyz" { + continue + } + for _, c := range block.ToolResult.Content { + if c.Video != nil { + got = c.Video + break + } + } + } + } + + require.NotNil(t, got, "expected toolResult.content[].video to round-trip; got nil") + assert.Equal(t, "mp4", got.Format) + require.NotNil(t, got.Source.Bytes) + assert.Equal(t, videoBytes, *got.Source.Bytes) + assert.Nil(t, got.Source.S3Location, "expected union member s3Location to be nil when bytes is set") +} + +// TestBedrockMixedBlockToolResultRoundTrip covers a toolResult.content array that +// mixes a representable block (text) with an unrepresentable one (searchResult). +// Because the envelope path triggers on *any* unrepresentable block and serializes +// the entire content array, the whole array is bundled and must be decoded back +// intact — both blocks, in order. This guards against the decode leg dropping the +// representable block (or vice-versa) when the two are interleaved. +func TestBedrockMixedBlockToolResultRoundTrip(t *testing.T) { + original := &bedrock.BedrockConverseRequest{ + ModelID: "anthropic.claude-sonnet-4-5", + Messages: []bedrock.BedrockMessage{ + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + {Text: schemas.Ptr("What is Apptio?")}, + }, + }, + { + Role: bedrock.BedrockMessageRoleAssistant, + Content: []bedrock.BedrockContentBlock{ + { + ToolUse: &bedrock.BedrockToolUse{ + ToolUseID: "tooluse_mixed_blocks", + Name: "RAGRequest", + Input: json.RawMessage(`{"query":"What is Apptio?"}`), + }, + }, + }, + }, + { + Role: bedrock.BedrockMessageRoleUser, + Content: []bedrock.BedrockContentBlock{ + { + ToolResult: &bedrock.BedrockToolResult{ + ToolUseID: "tooluse_mixed_blocks", + Status: schemas.Ptr("success"), + Content: []bedrock.BedrockContentBlock{ + {Text: schemas.Ptr("Summary: Apptio is a Bedrock passthrough customer.")}, + { + SearchResult: &bedrock.BedrockSearchResultBlock{ + Source: "Great Source of Information About Apptio", + Title: "12adbd74-46bd-4a88-88b2-0048755f6eb5", + Content: []bedrock.BedrockSearchResultContent{ + {Text: "Apptio is a company that makes calls to Bedrock using passthrough APIs via Bifrost"}, + }, + Citations: &bedrock.BedrockCitationsConfig{Enabled: true}, + }, + }, + }, + }, + }, + }, + }, + }, + System: []bedrock.BedrockSystemMessage{ + {Text: schemas.Ptr("Do not rely on your knowledge to answer. Use only the tool results.")}, + }, + } + + ctx := schemas.NewBifrostContext(context.Background(), schemas.NoDeadline) + + bifrostReq, err := original.ToBifrostResponsesRequest(ctx) + require.NoError(t, err) + require.NotNil(t, bifrostReq) + + rebuilt, err := bedrock.ToBedrockResponsesRequest(ctx, bifrostReq) + require.NoError(t, err) + require.NotNil(t, rebuilt) + + // Locate the rebuilt toolResult content array for our toolUseId. + var gotContent []bedrock.BedrockContentBlock + for _, msg := range rebuilt.Messages { + for _, block := range msg.Content { + if block.ToolResult == nil || block.ToolResult.ToolUseID != "tooluse_mixed_blocks" { + continue + } + gotContent = block.ToolResult.Content + } + } + + require.NotNil(t, gotContent, "expected toolResult with our toolUseId to round-trip; got nil") + require.Len(t, gotContent, 2, "expected both the text and searchResult blocks to survive the round-trip") + + // Order is preserved: the envelope is a JSON array, so block[0] is the text + // block and block[1] is the searchResult block. + require.NotNil(t, gotContent[0].Text, "expected first block to remain a text block") + assert.Equal(t, "Summary: Apptio is a Bedrock passthrough customer.", *gotContent[0].Text) + assert.Nil(t, gotContent[0].SearchResult, "text block must not gain a searchResult") + + got := gotContent[1].SearchResult + require.NotNil(t, got, "expected second block to remain a searchResult block") + assert.Nil(t, gotContent[1].Text, "searchResult block must not gain a text field") + assert.Equal(t, "Great Source of Information About Apptio", got.Source) + assert.Equal(t, "12adbd74-46bd-4a88-88b2-0048755f6eb5", got.Title) + require.Len(t, got.Content, 1) + assert.Equal(t, "Apptio is a company that makes calls to Bedrock using passthrough APIs via Bifrost", got.Content[0].Text) + require.NotNil(t, got.Citations) + assert.True(t, got.Citations.Enabled) +} diff --git a/core/providers/bedrock/responses.go b/core/providers/bedrock/responses.go index fcb439ced43..812fd0af92d 100644 --- a/core/providers/bedrock/responses.go +++ b/core/providers/bedrock/responses.go @@ -3163,7 +3163,12 @@ func ConvertBifrostMessagesToBedrockMessages(ctx context.Context, bifrostMessage // Convert result content to Bedrock format if msg.ResponsesToolMessage.Output != nil { if msg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr != nil { - resultContent = append(resultContent, tryParseJSONIntoContentBlock(*msg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr)) + outputStr := *msg.ResponsesToolMessage.Output.ResponsesToolCallOutputStr + if blocks, ok := decodeBedrockToolResultEnvelope(outputStr); ok { + resultContent = append(resultContent, blocks...) + } else { + resultContent = append(resultContent, tryParseJSONIntoContentBlock(outputStr)) + } } else if msg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks != nil { // Handle structured output blocks for _, block := range msg.ResponsesToolMessage.Output.ResponsesFunctionToolCallOutputBlocks { @@ -3989,9 +3994,24 @@ func convertSingleBedrockMessageToBifrostMessages(ctx *schemas.BifrostContext, m } else if block.ToolResult != nil { // Tool result content - typically not in assistant output but handled for completeness - // Prefer JSON payloads without unmarshalling; fallback to text + // Prefer JSON payloads without unmarshalling; fallback to text. + // If the content contains a searchResult (or any other block Bifrost's intermediate + // can't model natively), serialize the full content array into a sentinel envelope + // so it round-trips losslessly via ToBedrockResponsesRequest. var resultContent string - if len(block.ToolResult.Content) > 0 { + hasUnrepresentableBlock := false + for _, c := range block.ToolResult.Content { + if c.SearchResult != nil || c.Video != nil { + hasUnrepresentableBlock = true + break + } + } + if hasUnrepresentableBlock { + if envelope, err := encodeBedrockToolResultEnvelope(block.ToolResult.Content); err == nil { + resultContent = envelope + } + } + if resultContent == "" && len(block.ToolResult.Content) > 0 { // JSON first (no unmarshal; just one marshal to string when present) for _, c := range block.ToolResult.Content { if c.JSON != nil { diff --git a/core/providers/bedrock/types.go b/core/providers/bedrock/types.go index 8bc26195e80..2376235d157 100644 --- a/core/providers/bedrock/types.go +++ b/core/providers/bedrock/types.go @@ -201,6 +201,9 @@ type BedrockContentBlock struct { // Image content Image *BedrockImageSource `json:"image,omitempty"` + // Video content + Video *BedrockVideoBlock `json:"video,omitempty"` + // Document content Document *BedrockDocumentSource `json:"document,omitempty"` @@ -219,6 +222,9 @@ type BedrockContentBlock struct { // For Tool Call Result content JSON json.RawMessage `json:"json,omitempty"` + // Search result content (only valid inside toolResult.content per AWS Converse API) + SearchResult *BedrockSearchResultBlock `json:"searchResult,omitempty"` + // Cache point for the content block CachePoint *BedrockCachePoint `json:"cachePoint,omitempty"` @@ -261,6 +267,28 @@ type BedrockDocumentSourceData struct { Text *string `json:"text,omitempty"` // Plain text content } +// BedrockVideoBlock represents a video content block. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_VideoBlock.html +type BedrockVideoBlock struct { + Format string `json:"format"` // Required: mkv|mov|mp4|webm|flv|mpeg|mpg|wmv|three_gp + Source BedrockVideoSource `json:"source"` // Required: video source (bytes or s3Location, union) +} + +// BedrockVideoSource is the source for a BedrockVideoBlock. +// This is a tagged union — exactly one of Bytes or S3Location should be set. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_VideoSource.html +type BedrockVideoSource struct { + Bytes *string `json:"bytes,omitempty"` // Optional: base64-encoded video bytes (<25MB) + S3Location *BedrockS3Location `json:"s3Location,omitempty"` // Optional: S3 location +} + +// BedrockS3Location represents a storage location in an Amazon S3 bucket. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_S3Location.html +type BedrockS3Location struct { + URI string `json:"uri"` // Required: object URI starting with s3:// + BucketOwner *string `json:"bucketOwner,omitempty"` // Optional: 12-digit AWS account ID if cross-account +} + // BedrockToolUse represents a tool use request type BedrockToolUse struct { ToolUseID string `json:"toolUseId"` // Required: Unique identifier for this tool use @@ -277,6 +305,27 @@ type BedrockToolResult struct { Type *string `json:"type,omitempty"` // Optional: result type e.g. "nova_code_interpreter_result" } +// BedrockSearchResultBlock represents a search result tool-result content block. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_SearchResultBlock.html +type BedrockSearchResultBlock struct { + Source string `json:"source"` // Required: source URL or identifier + Title string `json:"title"` // Required: descriptive title + Content []BedrockSearchResultContent `json:"content"` // Required: search result content blocks + Citations *BedrockCitationsConfig `json:"citations,omitempty"` // Optional: citations config +} + +// BedrockSearchResultContent represents a single content block inside a SearchResult. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_SearchResultContentBlock.html +type BedrockSearchResultContent struct { + Text string `json:"text"` // Required: actual text content +} + +// BedrockCitationsConfig represents citations configuration for a search result. +// See: https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_CitationsConfig.html +type BedrockCitationsConfig struct { + Enabled bool `json:"enabled"` // Required: whether citations are enabled +} + // BedrockGuardContent represents guard content for guardrails type BedrockGuardContent struct { Text *BedrockGuardContentText `json:"text,omitempty"` diff --git a/core/providers/bedrock/utils.go b/core/providers/bedrock/utils.go index b90c224126c..59781bf4f20 100644 --- a/core/providers/bedrock/utils.go +++ b/core/providers/bedrock/utils.go @@ -10,6 +10,7 @@ import ( "regexp" "strings" + "github.com/bytedance/sonic" "github.com/tidwall/sjson" "github.com/maximhq/bifrost/core/providers/anthropic" @@ -2118,6 +2119,45 @@ func bedrockExtractFloat64(v interface{}) (float64, bool) { } } +// bedrockToolResultEnvelopeKey marks a sentinel-wrapped JSON string that carries a full +// BedrockToolResult.Content array through Bifrost's intermediate format. Used when the +// content includes blocks (e.g. searchResult) that the intermediate cannot model natively, +// so they round-trip losslessly on the Bedrock-native passthrough endpoint. +const bedrockToolResultEnvelopeKey = "__bifrost_bedrock_tool_result_content__" + +// encodeBedrockToolResultEnvelope serializes a BedrockToolResult.Content array into a +// sentinel-wrapped JSON object that decodeBedrockToolResultEnvelope can recover. +func encodeBedrockToolResultEnvelope(content []BedrockContentBlock) (string, error) { + envelope := map[string]any{bedrockToolResultEnvelopeKey: content} + b, err := sonic.Marshal(envelope) + if err != nil { + return "", err + } + return string(b), nil +} + +// decodeBedrockToolResultEnvelope is the inverse of encodeBedrockToolResultEnvelope. +// Returns (blocks, true) if s is a sentinel-wrapped tool-result envelope; (nil, false) otherwise. +// Non-envelope strings are returned untouched so the caller can fall through to tryParseJSONIntoContentBlock. +func decodeBedrockToolResultEnvelope(s string) ([]BedrockContentBlock, bool) { + if len(s) == 0 || s[0] != '{' || !strings.Contains(s, bedrockToolResultEnvelopeKey) { + return nil, false + } + var envelope map[string]json.RawMessage + if err := sonic.UnmarshalString(s, &envelope); err != nil { + return nil, false + } + raw, ok := envelope[bedrockToolResultEnvelopeKey] + if !ok || len(envelope) != 1 { + return nil, false + } + var blocks []BedrockContentBlock + if err := sonic.Unmarshal(raw, &blocks); err != nil { + return nil, false + } + return blocks, true +} + // tryParseJSONIntoContentBlock try to parse input text into a JSON and returns a proper // BedrockContentBlock based on the result. func tryParseJSONIntoContentBlock(text string) BedrockContentBlock { From bff5db085b8282c0692e71787e95a5dc17f983b8 Mon Sep 17 00:00:00 2001 From: roroghost17 Date: Fri, 22 May 2026 13:39:56 +0530 Subject: [PATCH 2/3] feat: starts forwarding x-bf-eh to OTEL --- core/bifrost.go | 24 ++++++++++++ core/schemas/trace.go | 91 +++++++++++++++++++++++++++++-------------- 2 files changed, 85 insertions(+), 30 deletions(-) diff --git a/core/bifrost.go b/core/bifrost.go index a696c152352..5a94914d0b5 100644 --- a/core/bifrost.go +++ b/core/bifrost.go @@ -5386,6 +5386,30 @@ func executeRequestWithRetries[T any]( } tracer.SetAttribute(handle, schemas.AttrNumberOfRetries, attempts) + // Surface caller-supplied extra headers (from x-bf-eh-* and direct-allowlist + // header forwarding) as span attributes so observability backends see the + // same set Bifrost forwards to the upstream provider. + if extraHeaders, ok := ctx.Value(schemas.BifrostContextKeyExtraHeaders).(map[string][]string); ok { + for name, values := range extraHeaders { + if name == "" || len(values) == 0 { + continue + } + // Never export credential-bearing headers verbatim. The transport + // layer denylists most sensitive headers, but plain authorization / + // set-cookie can still reach here, and core SDK callers bypass that + // guard entirely. Keep the key (presence is useful) but redact the value. + if schemas.IsSensitiveHeader(name) { + tracer.SetAttribute(handle, schemas.AttrExtraHeaderPrefix+name, schemas.RedactedAttrValue) + continue + } + if len(values) == 1 { + tracer.SetAttribute(handle, schemas.AttrExtraHeaderPrefix+name, values[0]) + } else { + tracer.SetAttribute(handle, schemas.AttrExtraHeaderPrefix+name, values) + } + } + } + // Populate LLM request attributes (messages, parameters, etc.) if req != nil { tracer.PopulateLLMRequestAttributes(handle, req) diff --git a/core/schemas/trace.go b/core/schemas/trace.go index 916285b8b49..654a3d4f054 100644 --- a/core/schemas/trace.go +++ b/core/schemas/trace.go @@ -2,22 +2,23 @@ package schemas import ( + "strings" "sync" "time" ) // Trace represents a distributed trace that captures the full lifecycle of a request type Trace struct { - RequestID string // Request ID for the trace - TraceID string // Unique identifier for this trace - ParentID string // Parent trace ID from incoming W3C traceparent header - RootSpan *Span // The root span of this trace - Spans []*Span // All spans in this trace - StartTime time.Time // When the trace started - EndTime time.Time // When the trace completed - Attributes map[string]any // Additional attributes for the trace + RequestID string // Request ID for the trace + TraceID string // Unique identifier for this trace + ParentID string // Parent trace ID from incoming W3C traceparent header + RootSpan *Span // The root span of this trace + Spans []*Span // All spans in this trace + StartTime time.Time // When the trace started + EndTime time.Time // When the trace completed + Attributes map[string]any // Additional attributes for the trace PluginLogs []PluginLogEntry // Plugin log entries accumulated during request processing - mu sync.Mutex // Mutex for thread-safe span operations + mu sync.Mutex // Mutex for thread-safe span operations } // AddSpan adds a span to the trace in a thread-safe manner @@ -257,21 +258,21 @@ const ( AttrOutputTokens = "gen_ai.usage.output_tokens" AttrUsageCost = "gen_ai.usage.cost" // Chat completion usage detail attributes - AttrPromptTokenDetailsText = "gen_ai.usage.prompt_token_details.text_tokens" - AttrPromptTokenDetailsAudio = "gen_ai.usage.prompt_token_details.audio_tokens" - AttrPromptTokenDetailsImage = "gen_ai.usage.prompt_token_details.image_tokens" + AttrPromptTokenDetailsText = "gen_ai.usage.prompt_token_details.text_tokens" + AttrPromptTokenDetailsAudio = "gen_ai.usage.prompt_token_details.audio_tokens" + AttrPromptTokenDetailsImage = "gen_ai.usage.prompt_token_details.image_tokens" AttrPromptTokenDetailsCachedRead = "gen_ai.usage.prompt_token_details.cached_read_tokens" AttrPromptTokenDetailsCachedWrite = "gen_ai.usage.prompt_token_details.cached_write_tokens" AttrPromptTokenDetailsCachedWrite5m = "gen_ai.usage.prompt_token_details.cached_write_tokens_5m" AttrPromptTokenDetailsCachedWrite1h = "gen_ai.usage.prompt_token_details.cached_write_tokens_1h" AttrCompletionTokenDetailsText = "gen_ai.usage.completion_token_details.text_tokens" - AttrCompletionTokenDetailsAudio = "gen_ai.usage.completion_token_details.audio_tokens" - AttrCompletionTokenDetailsImage = "gen_ai.usage.completion_token_details.image_tokens" - AttrCompletionTokenDetailsReason = "gen_ai.usage.completion_token_details.reasoning_tokens" - AttrCompletionTokenDetailsAccept = "gen_ai.usage.completion_token_details.accepted_prediction_tokens" - AttrCompletionTokenDetailsReject = "gen_ai.usage.completion_token_details.rejected_prediction_tokens" - AttrCompletionTokenDetailsCite = "gen_ai.usage.completion_token_details.citation_tokens" - AttrCompletionTokenDetailsSearch = "gen_ai.usage.completion_token_details.num_search_queries" + AttrCompletionTokenDetailsAudio = "gen_ai.usage.completion_token_details.audio_tokens" + AttrCompletionTokenDetailsImage = "gen_ai.usage.completion_token_details.image_tokens" + AttrCompletionTokenDetailsReason = "gen_ai.usage.completion_token_details.reasoning_tokens" + AttrCompletionTokenDetailsAccept = "gen_ai.usage.completion_token_details.accepted_prediction_tokens" + AttrCompletionTokenDetailsReject = "gen_ai.usage.completion_token_details.rejected_prediction_tokens" + AttrCompletionTokenDetailsCite = "gen_ai.usage.completion_token_details.citation_tokens" + AttrCompletionTokenDetailsSearch = "gen_ai.usage.completion_token_details.num_search_queries" // Error Attributes AttrError = "gen_ai.error" @@ -300,6 +301,9 @@ const ( AttrNumberOfRetries = "gen_ai.number_of_retries" AttrFallbackIndex = "gen_ai.fallback_index" + // Extra Header Attributes + AttrExtraHeaderPrefix = "gen_ai.request.extra_header." + // Responses API Request Attributes AttrPromptCacheKey = "gen_ai.request.prompt_cache_key" AttrReasoningEffort = "gen_ai.request.reasoning_effort" @@ -378,19 +382,19 @@ const ( AttrInputTokenDetailsAudio = "gen_ai.usage.input_token_details.audio_tokens" // Responses API usage detail attributes - AttrInputTokenDetailsImage = "gen_ai.usage.input_token_details.image_tokens" - AttrInputTokenDetailsCachedRead = "gen_ai.usage.input_token_details.cached_read_tokens" - AttrInputTokenDetailsCachedWrite = "gen_ai.usage.input_token_details.cached_write_tokens" + AttrInputTokenDetailsImage = "gen_ai.usage.input_token_details.image_tokens" + AttrInputTokenDetailsCachedRead = "gen_ai.usage.input_token_details.cached_read_tokens" + AttrInputTokenDetailsCachedWrite = "gen_ai.usage.input_token_details.cached_write_tokens" AttrInputTokenDetailsCachedWrite5m = "gen_ai.usage.input_token_details.cached_write_tokens_5m" AttrInputTokenDetailsCachedWrite1h = "gen_ai.usage.input_token_details.cached_write_tokens_1h" - AttrOutputTokenDetailsText = "gen_ai.usage.output_token_details.text_tokens" - AttrOutputTokenDetailsAudio = "gen_ai.usage.output_token_details.audio_tokens" - AttrOutputTokenDetailsImage = "gen_ai.usage.output_token_details.image_tokens" - AttrOutputTokenDetailsReason = "gen_ai.usage.output_token_details.reasoning_tokens" - AttrOutputTokenDetailsAccept = "gen_ai.usage.output_token_details.accepted_prediction_tokens" - AttrOutputTokenDetailsReject = "gen_ai.usage.output_token_details.rejected_prediction_tokens" - AttrOutputTokenDetailsCite = "gen_ai.usage.output_token_details.citation_tokens" - AttrOutputTokenDetailsSearch = "gen_ai.usage.output_token_details.num_search_queries" + AttrOutputTokenDetailsText = "gen_ai.usage.output_token_details.text_tokens" + AttrOutputTokenDetailsAudio = "gen_ai.usage.output_token_details.audio_tokens" + AttrOutputTokenDetailsImage = "gen_ai.usage.output_token_details.image_tokens" + AttrOutputTokenDetailsReason = "gen_ai.usage.output_token_details.reasoning_tokens" + AttrOutputTokenDetailsAccept = "gen_ai.usage.output_token_details.accepted_prediction_tokens" + AttrOutputTokenDetailsReject = "gen_ai.usage.output_token_details.rejected_prediction_tokens" + AttrOutputTokenDetailsCite = "gen_ai.usage.output_token_details.citation_tokens" + AttrOutputTokenDetailsSearch = "gen_ai.usage.output_token_details.num_search_queries" // File Operation Attributes AttrFileID = "gen_ai.file.id" @@ -410,3 +414,30 @@ const ( AttrFileAfter = "gen_ai.file.after" AttrFileOrder = "gen_ai.file.order" ) + +// RedactedAttrValue is the placeholder recorded in place of a sensitive header +// value, following the OpenTelemetry HTTP semantic-convention guidance for +// redacting credentials. +const RedactedAttrValue = "REDACTED" + +// IsSensitiveHeader reports whether a header name carries credentials that must +// not be exported verbatim into span attributes. The match is case-insensitive +// and trims surrounding whitespace so callers using the core SDK directly (which +// bypass the transport-layer security denylist) are still protected. Beyond the +// well-known exact names, substring/suffix patterns catch credential-bearing +// variants like x-auth-token, x-amz-security-token, and provider-specific +// *-api-key headers. +func IsSensitiveHeader(name string) bool { + normalized := strings.ToLower(strings.TrimSpace(name)) + + switch normalized { + case "authorization", "proxy-authorization", "cookie", "set-cookie": + return true + } + + return strings.Contains(normalized, "api-key") || + strings.Contains(normalized, "authorization") || + strings.Contains(normalized, "secret") || + strings.HasSuffix(normalized, "-token") || + strings.HasSuffix(normalized, "_token") +} From 930e60e0820c2a71aec5e8d0a17face0905af8fb Mon Sep 17 00:00:00 2001 From: roroghost17 Date: Fri, 22 May 2026 14:07:30 +0530 Subject: [PATCH 3/3] docs: adds docs for OTEL on OSS features list and examples --- docs/docs.json | 1 + docs/features/observability/otel.mdx | 6 ++ docs/features/otel.mdx | 95 ++++++++++++++++++++++++++++ docs/providers/request-options.mdx | 20 ++++++ 4 files changed, 122 insertions(+) create mode 100644 docs/features/otel.mdx diff --git a/docs/docs.json b/docs/docs.json index 2ea11332b69..ec7e725d656 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -221,6 +221,7 @@ ] }, "features/telemetry", + "features/otel", "features/semantic-caching", { "group": "Prompt Repository", diff --git a/docs/features/observability/otel.mdx b/docs/features/observability/otel.mdx index 5459aed2bfc..2eb665f8e65 100644 --- a/docs/features/observability/otel.mdx +++ b/docs/features/observability/otel.mdx @@ -664,6 +664,12 @@ Each trace includes comprehensive LLM operation metadata following OpenTelemetry - Latency and timing (start/end timestamps) - Error details with status codes +### Caller-Supplied Headers + +Headers Bifrost forwards to the upstream provider — both `x-bf-eh-*` prefixed headers and headers matched by the [direct allowlist](/deployment-guides/config-json/client#header-filtering) — are also surfaced on the `llm.call` span as `gen_ai.request.extra_header.` attributes. This makes it easy to filter or correlate traces by caller context (session ID, tenant ID, correlation IDs) without standing up extra plumbing. + +For example, sending `x-bf-eh-session-id: sess-abc-123` produces the span attribute `gen_ai.request.extra_header.session-id = "sess-abc-123"`. See [Extra Headers](/providers/request-options#extra-headers-x-bf-eh) for the full request format. + ### Example Span ```json diff --git a/docs/features/otel.mdx b/docs/features/otel.mdx new file mode 100644 index 00000000000..835de957036 --- /dev/null +++ b/docs/features/otel.mdx @@ -0,0 +1,95 @@ +--- +title: "OpenTelemetry" +description: "Native OpenTelemetry tracing for every LLM call routed through Bifrost — export to any OTLP collector." +icon: "telescope" +--- + +## Overview + +Bifrost provides built-in OpenTelemetry support that emits a fully-attributed span for every LLM request, including retries and fallbacks. Spans follow the [OpenTelemetry GenAI semantic conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), so they correlate cleanly with the rest of your application traces and ship over OTLP to any compatible backend — Grafana, Datadog, New Relic, Honeycomb, Langfuse, Arize Phoenix, or a self-hosted OTel Collector. + +**Key Features:** + +- **Native OTLP Export** — HTTP or gRPC transport to any OTLP-compatible backend, no vendor lock-in +- **GenAI Semantic Conventions** — `gen_ai.*` attributes for provider, model, tokens, cost +- **Per-Attempt Spans** — retries and fallbacks each get their own span with their own context +- **Streaming-Aware** — accumulates chunks and emits one complete span per request +- **Cost Tracking** — `gen_ai.usage.cost` attribute computed from the model catalog on every call +- **Dynamic Attributes** — runtime span enrichment via `x-bf-eh-*` headers +- **Push Metrics for Clusters** — OTLP-based metrics export for multi-node deployments +- **Configurable Span Granularity** — include or exclude plugin pre/post-hook spans +- **Async Emission** — zero impact on request latency + +OpenTelemetry export runs asynchronously to ensure span emission doesn't impact request latency or throughput. + +--- + +## Captured Attributes + +Every LLM call produces a span whose attributes follow OpenTelemetry GenAI semantic conventions. The main categories: + +| Category | Example attributes | +|----------|--------------------| +| **Provider & model** | `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model` | +| **Request parameters** | `gen_ai.request.temperature`, `gen_ai.request.max_tokens`, `gen_ai.request.top_p` | +| **Usage & cost** | `gen_ai.usage.prompt_tokens`, `gen_ai.usage.completion_tokens`, `gen_ai.usage.cost` | +| **Bifrost context** | `gen_ai.virtual_key_id`, `gen_ai.team_id`, `gen_ai.customer_id`, `gen_ai.number_of_retries` | +| **Caller-supplied headers** | `gen_ai.request.extra_header.` (see below) | +| **Input / output** | `gen_ai.input.messages`, `gen_ai.output.messages` | + +See [OpenTelemetry → Captured Data](/features/observability/otel#captured-data) for the full attribute list and an example span payload. + +--- + +## Dynamic Attribute Injection + +Any header Bifrost forwards to the upstream provider is also surfaced on the `llm.call` span. This includes: + +- `x-bf-eh-*` prefixed extra headers (the standard mechanism) +- Headers matched by the [direct allowlist](/deployment-guides/config-json/client#header-filtering) (e.g. `anthropic-beta`) + +For `x-bf-eh-*` headers the `x-bf-eh-` prefix is stripped and the remainder is lowercased; for direct-allowlist headers the header name is used as-is (lowercased). The result becomes the `` in `gen_ai.request.extra_header.`. The same security denylist and filter config that gates provider forwarding gates the span attribute — they are always the same set. + +### Example: forwarding a session ID + +```bash +curl -X POST http://localhost:8080/v1/chat/completions \ + -H 'Content-Type: application/json' \ + -H 'x-bf-eh-session-id: sess-abc-123' \ + -H 'x-bf-eh-tenant-id: acme-corp' \ + -d '{ + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +Result on the `llm.call` span: + +| Attribute | Value | +|-----------|-------| +| `gen_ai.request.extra_header.session-id` | `sess-abc-123` | +| `gen_ai.request.extra_header.tenant-id` | `acme-corp` | + +You can then filter or group traces by session in Grafana, Datadog, Honeycomb, Langfuse, etc. — no extra wiring required. + + +Want runtime labels on Prometheus metrics instead of OTel spans? Use `x-bf-dim-*` headers — see [Telemetry → Dynamic Label Injection](./telemetry#dynamic-label-injection). The same `x-bf-dim-*` values also flow through to OTel as span attributes. + + +--- + +## Setup + +OpenTelemetry export is configured through the Bifrost UI, `config.json`, or the Go SDK. Full configuration options, popular platform recipes (Grafana Cloud, Datadog, New Relic, Honeycomb, Langfuse, self-hosted), cluster-mode metrics push, and the local Docker Compose stack are documented on the integrations page: + + + Full configuration reference, platform-specific examples, Docker Compose stack, and metrics push setup. + + +--- + +## Next Steps + +- **[OpenTelemetry Integration](./observability/otel)** — Full setup with platform-specific examples +- **[Telemetry](./telemetry)** — Prometheus metrics that complement OTel traces +- **[Extra Headers Reference](/providers/request-options#extra-headers-x-bf-eh)** — Full `x-bf-eh-*` request format diff --git a/docs/providers/request-options.mdx b/docs/providers/request-options.mdx index 515052066b7..d8629c3d6df 100644 --- a/docs/providers/request-options.mdx +++ b/docs/providers/request-options.mdx @@ -687,12 +687,32 @@ Input: messages, The headers `x-bf-eh-user-id` and `x-bf-eh-tracking-id` are forwarded to the provider as `user-id` and `tracking-id` respectively. +When the OTel plugin is enabled, each forwarded header is also attached to the `llm.call` span as a `gen_ai.request.extra_header.` attribute, so the same metadata you send to the provider is searchable in your observability backend without extra wiring. + **Example use cases:** - User identification: `x-bf-eh-user-id`, `x-bf-eh-tenant-id` - Request tracking: `x-bf-eh-correlation-id`, `x-bf-eh-trace-id` - Custom metadata: `x-bf-eh-department`, `x-bf-eh-cost-center` - A/B testing: `x-bf-eh-experiment-id`, `x-bf-eh-variant` +**Example: forwarding a session ID** + +Send a stable `session-id` for each user session so every LLM call shares the same value on both the provider request and the OTel span: + +```bash +curl --location 'http://localhost:8080/v1/chat/completions' \ +--header 'x-bf-eh-session-id: sess-abc-123' \ +--header 'Content-Type: application/json' \ +--data '{ + "model": "openai/gpt-4o-mini", + "messages": [{"role": "user", "content": "Hello!"}] +}' +``` + +Result: +- Provider receives `session-id: sess-abc-123` in the request headers. +- OTel `llm.call` span gets attribute `gen_ai.request.extra_header.session-id = "sess-abc-123"` so you can filter or group traces by session in Grafana, Datadog, Honeycomb, Langfuse, etc. + ## Semantic Cache Options These options control semantic caching behavior.