diff --git a/core/schemas/redaction.go b/core/schemas/redaction.go index e538d2387ef..b08a732a23e 100644 --- a/core/schemas/redaction.go +++ b/core/schemas/redaction.go @@ -6,10 +6,57 @@ import ( "strings" ) +// RedactionPhase identifies which request lifecycle phase produced a redaction finding. +type RedactionPhase string + +const ( + // RedactionPhaseInput marks redaction findings discovered while inspecting request-side content. + RedactionPhaseInput RedactionPhase = "input" + // RedactionPhaseOutput marks redaction findings discovered while inspecting response-side content. + RedactionPhaseOutput RedactionPhase = "output" +) + // RedactionData carries request-scoped redaction data from guardrails to log and trace sinks. type RedactionData struct { - ReversibleMappings map[string]string `json:"reversible_mappings,omitempty"` - LiteralReplacements map[string]string `json:"literal_replacements,omitempty"` + LiteralReplacements RedactionMapsByPhase `json:"literal_replacements,omitempty"` + ReversibleMappings RedactionMapsByPhase `json:"reversible_mappings,omitempty"` +} + +// RedactionMapsByPhase stores replacement maps separately for request and response content. +type RedactionMapsByPhase struct { + Input map[string]string `json:"input,omitempty"` + Output map[string]string `json:"output,omitempty"` +} + +// Clone returns an owned copy of the phase-scoped maps. +func (m RedactionMapsByPhase) Clone() RedactionMapsByPhase { + return RedactionMapsByPhase{ + Input: maps.Clone(m.Input), + Output: maps.Clone(m.Output), + } +} + +// HasReplacements reports whether either phase has replacement entries. +func (m RedactionMapsByPhase) HasReplacements() bool { + return len(m.Input) > 0 || len(m.Output) > 0 +} + +// MergePhase merges replacements into one phase, copying entries so callers cannot mutate stored state. +func (m *RedactionMapsByPhase) MergePhase(phase RedactionPhase, replacements map[string]string) { + if m == nil || len(replacements) == 0 { + return + } + switch phase { + case RedactionPhaseInput: + m.Input = mergeRedactionStringMaps(m.Input, replacements) + case RedactionPhaseOutput: + m.Output = mergeRedactionStringMaps(m.Output, replacements) + } +} + +// MergedForMixedFields returns both phase maps for fields that can contain input and output. +func (m RedactionMapsByPhase) MergedForMixedFields() map[string]string { + return mergeRedactionStringMaps(m.Input, m.Output) } // Clone returns an owned snapshot of the redaction data maps. @@ -21,18 +68,23 @@ type RedactionData struct { // contain immutable strings. func (d RedactionData) Clone() RedactionData { return RedactionData{ - ReversibleMappings: maps.Clone(d.ReversibleMappings), - LiteralReplacements: maps.Clone(d.LiteralReplacements), + LiteralReplacements: d.LiteralReplacements.Clone(), + ReversibleMappings: d.ReversibleMappings.Clone(), } } +// HasReplacements reports whether any reversible or literal redaction data is present. +func (d RedactionData) HasReplacements() bool { + return d.LiteralReplacements.HasReplacements() || d.ReversibleMappings.HasReplacements() +} + // RedactionDataFromContext returns the redaction data stored on ctx. func RedactionDataFromContext(ctx *BifrostContext) (RedactionData, bool) { if ctx == nil { return RedactionData{}, false } data, ok := ctx.Value(BifrostContextKeyRedactionData).(RedactionData) - if !ok || (len(data.ReversibleMappings) == 0 && len(data.LiteralReplacements) == 0) { + if !ok || !data.HasReplacements() { return RedactionData{}, false } return data, true @@ -40,7 +92,7 @@ func RedactionDataFromContext(ctx *BifrostContext) (RedactionData, bool) { // SetRedactionDataOnContext stores non-empty redaction data on ctx. func SetRedactionDataOnContext(ctx *BifrostContext, data RedactionData) bool { - if ctx == nil || (len(data.ReversibleMappings) == 0 && len(data.LiteralReplacements) == 0) { + if ctx == nil || !data.HasReplacements() { return false } ctx.SetValue(BifrostContextKeyRedactionData, data) @@ -49,23 +101,7 @@ func SetRedactionDataOnContext(ctx *BifrostContext, data RedactionData) bool { // IsContentAttribute reports whether a span attribute may carry user or model content. func IsContentAttribute(key string) bool { - switch key { - case AttrInputMessages, AttrOutputMessages, - AttrInputText, AttrInputSpeech, - AttrInputEmbedding, - AttrPrompt, AttrInstructions, - AttrRespReasoningText: - return true - case AttrTools, AttrRespTools, - AttrToolName, AttrToolCallID, - AttrToolCallArguments, AttrToolCallResult, - AttrToolType, - AttrToolChoiceType, AttrToolChoiceName, - AttrRespToolChoiceType, AttrRespToolChoiceName: - return true - default: - return false - } + return traceContentAttributeScopeForKey(key) != traceContentAttributeScopeNone } // ApplyLiteralReplacements performs deterministic best-effort string redaction. @@ -130,3 +166,18 @@ func RedactAttributeValue(value any, replacements map[string]string) any { return value } } + +// mergeRedactionStringMaps returns a copied map containing both map values, with next taking precedence. +func mergeRedactionStringMaps(current map[string]string, next map[string]string) map[string]string { + if len(current) == 0 && len(next) == 0 { + return nil + } + merged := maps.Clone(current) + if merged == nil { + merged = make(map[string]string, len(next)) + } + for key, value := range next { + merged[key] = value + } + return merged +} diff --git a/core/schemas/redaction_test.go b/core/schemas/redaction_test.go index 2a52e23867a..740fc6ba07e 100644 --- a/core/schemas/redaction_test.go +++ b/core/schemas/redaction_test.go @@ -14,8 +14,14 @@ import ( func TestRedactionDataContextRoundTrip(t *testing.T) { ctx := NewBifrostContext(context.Background(), NoDeadline) data := RedactionData{ - ReversibleMappings: map[string]string{"EMAIL-1": "alex@example.com"}, - LiteralReplacements: map[string]string{"alex@example.com": "[EMAIL-1]"}, + LiteralReplacements: RedactionMapsByPhase{ + Input: map[string]string{"alex@example.com": "[EMAIL-1]"}, + Output: map[string]string{"rivera@example.com": "[EMAIL-2]"}, + }, + ReversibleMappings: RedactionMapsByPhase{ + Input: map[string]string{"EMAIL-1": "alex@example.com"}, + Output: map[string]string{"EMAIL-2": "rivera@example.com"}, + }, } require.True(t, SetRedactionDataOnContext(ctx, data)) @@ -28,16 +34,26 @@ func TestRedactionDataContextRoundTrip(t *testing.T) { // TestRedactionDataCloneCopiesMaps verifies clones do not share mutable map storage. func TestRedactionDataCloneCopiesMaps(t *testing.T) { data := RedactionData{ - ReversibleMappings: map[string]string{"EMAIL-1": "alex@example.com"}, - LiteralReplacements: map[string]string{"alex@example.com": "[EMAIL-1]"}, + LiteralReplacements: RedactionMapsByPhase{ + Input: map[string]string{"alex@example.com": "[EMAIL-1]"}, + Output: map[string]string{"rivera@example.com": "[EMAIL-2]"}, + }, + ReversibleMappings: RedactionMapsByPhase{ + Input: map[string]string{"EMAIL-1": "alex@example.com"}, + Output: map[string]string{"EMAIL-2": "rivera@example.com"}, + }, } clone := data.Clone() - data.ReversibleMappings["EMAIL-1"] = "mutated@example.com" - data.LiteralReplacements["alex@example.com"] = "[MUTATED]" - - assert.Equal(t, "alex@example.com", clone.ReversibleMappings["EMAIL-1"]) - assert.Equal(t, "[EMAIL-1]", clone.LiteralReplacements["alex@example.com"]) + data.ReversibleMappings.Input["EMAIL-1"] = "mutated@example.com" + data.ReversibleMappings.Output["EMAIL-2"] = "mutated@example.com" + data.LiteralReplacements.Input["alex@example.com"] = "[MUTATED]" + data.LiteralReplacements.Output["rivera@example.com"] = "[MUTATED]" + + assert.Equal(t, "alex@example.com", clone.ReversibleMappings.Input["EMAIL-1"]) + assert.Equal(t, "rivera@example.com", clone.ReversibleMappings.Output["EMAIL-2"]) + assert.Equal(t, "[EMAIL-1]", clone.LiteralReplacements.Input["alex@example.com"]) + assert.Equal(t, "[EMAIL-2]", clone.LiteralReplacements.Output["rivera@example.com"]) } // TestRedactionDataFromContextRejectsSerializedValues verifies the handoff remains typed. @@ -77,7 +93,7 @@ func TestIsContentAttribute(t *testing.T) { assert.False(t, IsContentAttribute(TraceAttrSessionID)) } -// TestTraceApplyRedactionReplacementsRedactsContentAttributes verifies trace redaction touches all spans. +// TestTraceApplyRedactionReplacementsRedactsContentAttributes verifies trace redaction honors attribute phase. func TestTraceApplyRedactionReplacementsRedactsContentAttributes(t *testing.T) { trace := &Trace{} root := &Span{} @@ -85,16 +101,17 @@ func TestTraceApplyRedactionReplacementsRedactsContentAttributes(t *testing.T) { trace.RootSpan = root trace.Spans = []*Span{root, child} - root.SetAttribute(AttrInputMessages, `{"content":"email alex@example.com"}`) + root.SetAttribute(AttrInputMessages, `{"content":"email alex@example.com and bob@example.com"}`) root.SetAttribute(AttrRequestModel, "alex@example.com") - child.SetAttribute(AttrOutputMessages, []string{"reply to alex@example.com"}) + child.SetAttribute(AttrOutputMessages, []string{"reply to alex@example.com and bob@example.com"}) child.SetAttribute(AttrToolCallArguments, map[string]any{ "customer": map[string]any{ "email": "alex@example.com", - "tags": []any{"safe", "alex@example.com"}, + "owner": "bob@example.com", + "tags": []any{"safe", "alex@example.com", "bob@example.com"}, }, "metadata": map[string]string{ - "owner": "alex@example.com", + "owner": "bob@example.com", }, "literal_key_alex@example.com": "key should redact too", "count": 42, @@ -102,32 +119,36 @@ func TestTraceApplyRedactionReplacementsRedactsContentAttributes(t *testing.T) { child.AddEvent(SpanEvent{ Name: "llm.message", Attributes: map[string]any{ - AttrInputMessages: `{"content":"event alex@example.com"}`, - AttrRequestModel: "alex@example.com", + AttrInputMessages: `{"content":"event alex@example.com and bob@example.com"}`, + AttrOutputMessages: `{"content":"event alex@example.com and bob@example.com"}`, + AttrRequestModel: "alex@example.com", }, }) - trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"bob@example.com": "[EMAIL-2]"}) trace.ApplyRedactionReplacements() - assert.Equal(t, `{"content":"email [EMAIL-1]"}`, root.Attributes[AttrInputMessages]) + assert.Equal(t, `{"content":"email [EMAIL-1] and bob@example.com"}`, root.Attributes[AttrInputMessages]) assert.Equal(t, "alex@example.com", root.Attributes[AttrRequestModel]) - assert.Equal(t, []string{"reply to [EMAIL-1]"}, child.Attributes[AttrOutputMessages]) + assert.Equal(t, []string{"reply to alex@example.com and [EMAIL-2]"}, child.Attributes[AttrOutputMessages]) assert.Equal(t, map[string]any{ "customer": map[string]any{ "email": "[EMAIL-1]", - "tags": []any{"safe", "[EMAIL-1]"}, + "owner": "[EMAIL-2]", + "tags": []any{"safe", "[EMAIL-1]", "[EMAIL-2]"}, }, "metadata": map[string]string{ - "owner": "[EMAIL-1]", + "owner": "[EMAIL-2]", }, "literal_key_[EMAIL-1]": "key should redact too", "count": 42, }, child.Attributes[AttrToolCallArguments]) require.Len(t, child.Events, 1) - assert.Equal(t, `{"content":"event [EMAIL-1]"}`, child.Events[0].Attributes[AttrInputMessages]) + assert.Equal(t, `{"content":"event [EMAIL-1] and bob@example.com"}`, child.Events[0].Attributes[AttrInputMessages]) + assert.Equal(t, `{"content":"event alex@example.com and [EMAIL-2]"}`, child.Events[0].Attributes[AttrOutputMessages]) assert.Equal(t, "alex@example.com", child.Events[0].Attributes[AttrRequestModel]) - assert.Nil(t, trace.redactionReplacements) + assert.False(t, trace.redactionReplacements.HasReplacements()) } // TestTraceSetRedactionReplacementsMergesCalls verifies input/output replacement windows accumulate. @@ -141,25 +162,28 @@ func TestTraceSetRedactionReplacementsMergesCalls(t *testing.T) { root.SetAttribute(AttrInputMessages, `{"content":"email input@example.com"}`) child.SetAttribute(AttrOutputMessages, `{"content":"reply output@example.com"}`) - trace.SetRedactionReplacements(map[string]string{"input@example.com": "[EMAIL-1]"}) - trace.SetRedactionReplacements(map[string]string{"output@example.com": "[EMAIL-2]"}) + trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"input@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"output@example.com": "[EMAIL-2]"}) trace.ApplyRedactionReplacements() assert.Equal(t, `{"content":"email [EMAIL-1]"}`, root.Attributes[AttrInputMessages]) assert.Equal(t, `{"content":"reply [EMAIL-2]"}`, child.Attributes[AttrOutputMessages]) - assert.Nil(t, trace.redactionReplacements) + assert.False(t, trace.redactionReplacements.HasReplacements()) } // TestTraceRedactionReplacementsDoNotSerialize verifies connector-facing replacements stay internal. func TestTraceRedactionReplacementsDoNotSerialize(t *testing.T) { trace := &Trace{TraceID: "trace-1"} - trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"rivera@example.com": "[EMAIL-2]"}) serialized, err := sonic.MarshalString(trace) require.NoError(t, err) assert.NotContains(t, serialized, "alex@example.com") assert.NotContains(t, serialized, "[EMAIL-1]") + assert.NotContains(t, serialized, "rivera@example.com") + assert.NotContains(t, serialized, "[EMAIL-2]") assert.False(t, strings.Contains(serialized, "redactionReplacements")) assert.False(t, strings.Contains(serialized, "RedactionReplacements")) } @@ -167,9 +191,10 @@ func TestTraceRedactionReplacementsDoNotSerialize(t *testing.T) { // TestTraceResetClearsRedactionReplacements verifies pooled traces cannot retain request redaction data. func TestTraceResetClearsRedactionReplacements(t *testing.T) { trace := &Trace{} - trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"}) + trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"rivera@example.com": "[EMAIL-2]"}) trace.Reset() - assert.Nil(t, trace.redactionReplacements) + assert.False(t, trace.redactionReplacements.HasReplacements()) } diff --git a/core/schemas/trace.go b/core/schemas/trace.go index 73cb887bf6f..6d74c0f7e9f 100644 --- a/core/schemas/trace.go +++ b/core/schemas/trace.go @@ -19,8 +19,8 @@ type Trace struct { Attributes map[string]any // Additional attributes for the trace RequestHeaders map[string]string // Lowercased request headers, populated only when a connector opts in PluginLogs []PluginLogEntry // Plugin log entries accumulated during request processing - redactionReplacements map[string]string // Raw-to-placeholder replacements; unexported so connectors never serialize the map - mu sync.Mutex // Mutex for thread-safe span operations + redactionReplacements RedactionMapsByPhase + mu sync.Mutex // Mutex for thread-safe span operations } // Trace-level attribute keys. Unlike span attributes, trace attributes are never @@ -76,7 +76,7 @@ func (t *Trace) SetRequestHeaders(headers map[string]string) { } // SetRedactionReplacements merges connector-facing raw-to-placeholder replacements on the trace. -func (t *Trace) SetRedactionReplacements(replacements map[string]string) { +func (t *Trace) SetRedactionReplacements(phase RedactionPhase, replacements map[string]string) { if len(replacements) == 0 { return } @@ -91,37 +91,28 @@ func (t *Trace) SetRedactionReplacements(replacements map[string]string) { } t.mu.Lock() defer t.mu.Unlock() - if t.redactionReplacements == nil { - t.redactionReplacements = make(map[string]string, len(copied)) - } - for raw, placeholder := range copied { - t.redactionReplacements[raw] = placeholder - } + t.redactionReplacements.MergePhase(phase, copied) } // ApplyRedactionReplacements redacts content attributes on every span in the trace and clears the replacement map. func (t *Trace) ApplyRedactionReplacements() { t.mu.Lock() - if len(t.redactionReplacements) == 0 { + if !t.redactionReplacements.HasReplacements() { t.mu.Unlock() return } - replacements := make(map[string]string, len(t.redactionReplacements)) - for raw, placeholder := range t.redactionReplacements { - replacements[raw] = placeholder - delete(t.redactionReplacements, raw) - } - t.redactionReplacements = nil + replacements := t.redactionReplacements.Clone() + t.redactionReplacements = RedactionMapsByPhase{} rootSpan := t.RootSpan spans := append([]*Span(nil), t.Spans...) t.mu.Unlock() - redactSpanAttributes(rootSpan, replacements) + redactSpanAttributes(rootSpan, replacements.Input, replacements.Output) for _, span := range spans { if span == nil || span == rootSpan { continue } - redactSpanAttributes(span, replacements) + redactSpanAttributes(span, replacements.Input, replacements.Output) } } @@ -163,10 +154,7 @@ func (t *Trace) Reset() { t.EndTime = time.Time{} t.Attributes = nil t.RequestHeaders = nil - for raw := range t.redactionReplacements { - delete(t.redactionReplacements, raw) - } - t.redactionReplacements = nil + t.redactionReplacements = RedactionMapsByPhase{} for i := range t.PluginLogs { t.PluginLogs[i] = PluginLogEntry{} } @@ -183,21 +171,67 @@ func (t *Trace) AppendPluginLogs(logs []PluginLogEntry) { t.mu.Unlock() } -// redactSpanAttributes applies trace redaction replacements to one span's content attributes. -func redactSpanAttributes(span *Span, replacements map[string]string) { - if span == nil || len(replacements) == 0 { +// traceContentAttributeScope describes which phase can safely redact a trace content attribute. +type traceContentAttributeScope int + +const ( + traceContentAttributeScopeNone traceContentAttributeScope = iota + traceContentAttributeScopeInput + traceContentAttributeScopeOutput + traceContentAttributeScopeMixed +) + +// traceRedactionReplacementsForAttribute selects the phase-specific replacements for one trace attribute. +func traceRedactionReplacementsForAttribute(key string, inputReplacements map[string]string, outputReplacements map[string]string) map[string]string { + switch traceContentAttributeScopeForKey(key) { + case traceContentAttributeScopeInput: + return inputReplacements + case traceContentAttributeScopeOutput: + return outputReplacements + case traceContentAttributeScopeMixed: + return mergeTraceRedactionReplacements(inputReplacements, outputReplacements) + default: + return nil + } +} + +// traceContentAttributeScopeForKey classifies content attributes by request/response phase. +func traceContentAttributeScopeForKey(key string) traceContentAttributeScope { + switch key { + case AttrInputMessages, AttrInputText, AttrInputSpeech, AttrInputEmbedding, + AttrPrompt, AttrInstructions, + AttrTools, AttrToolChoiceType, AttrToolChoiceName, + AttrRespTools, AttrRespToolChoiceType, AttrRespToolChoiceName: + return traceContentAttributeScopeInput + case AttrOutputMessages, AttrRespReasoningText: + return traceContentAttributeScopeOutput + case AttrToolName, AttrToolCallID, AttrToolCallArguments, AttrToolCallResult, AttrToolType: + return traceContentAttributeScopeMixed + default: + return traceContentAttributeScopeNone + } +} + +// mergeTraceRedactionReplacements returns a combined replacement map for mixed trace attributes. +func mergeTraceRedactionReplacements(inputReplacements map[string]string, outputReplacements map[string]string) map[string]string { + return mergeRedactionStringMaps(inputReplacements, outputReplacements) +} + +// redactSpanAttributes applies phase-aware trace redaction replacements to one span's content attributes. +func redactSpanAttributes(span *Span, inputReplacements map[string]string, outputReplacements map[string]string) { + if span == nil || (len(inputReplacements) == 0 && len(outputReplacements) == 0) { return } span.mu.Lock() defer span.mu.Unlock() for key, value := range span.Attributes { - if IsContentAttribute(key) { + if replacements := traceRedactionReplacementsForAttribute(key, inputReplacements, outputReplacements); len(replacements) > 0 { span.Attributes[key] = RedactAttributeValue(value, replacements) } } for i := range span.Events { for key, value := range span.Events[i].Attributes { - if IsContentAttribute(key) { + if replacements := traceRedactionReplacementsForAttribute(key, inputReplacements, outputReplacements); len(replacements) > 0 { span.Events[i].Attributes[key] = RedactAttributeValue(value, replacements) } } diff --git a/core/schemas/tracer.go b/core/schemas/tracer.go index a28062a1023..3a1a006a4b0 100644 --- a/core/schemas/tracer.go +++ b/core/schemas/tracer.go @@ -173,8 +173,8 @@ type Tracer interface { // Thread-safe. Should be called after plugin hooks complete, before trace completion. AttachPluginLogs(traceID string, logs []PluginLogEntry) - // SetTraceRedactionReplacements stores connector-facing raw-to-placeholder replacements on a trace. - SetTraceRedactionReplacements(traceID string, replacements map[string]string) + // SetTraceRedactionReplacements stores phase-scoped connector-facing replacements on a trace. + SetTraceRedactionReplacements(traceID string, phase RedactionPhase, replacements map[string]string) // CompleteAndFlushTrace ends a trace, exports it to observability plugins, and // releases the trace resources. Used by transports that bypass normal HTTP trace completion. @@ -294,7 +294,7 @@ func (n *NoOpTracer) GateSend(_ string, chunk *BifrostStreamChunk, _ bool, _ boo func (n *NoOpTracer) AttachPluginLogs(_ string, _ []PluginLogEntry) {} // SetTraceRedactionReplacements does nothing. -func (n *NoOpTracer) SetTraceRedactionReplacements(_ string, _ map[string]string) {} +func (n *NoOpTracer) SetTraceRedactionReplacements(_ string, _ RedactionPhase, _ map[string]string) {} // CompleteAndFlushTrace does nothing. func (n *NoOpTracer) CompleteAndFlushTrace(_ string) {} diff --git a/docs/enterprise/guardrails/redaction.mdx b/docs/enterprise/guardrails/redaction.mdx index b88987a8fee..a90e78df42f 100644 --- a/docs/enterprise/guardrails/redaction.mdx +++ b/docs/enterprise/guardrails/redaction.mdx @@ -117,8 +117,12 @@ Users need the `Logs:Reveal` permission to reveal original values for a log that ```json { "redaction_mapping": { - "EMAIL-1": "alex@example.com", - "PHONE_NUMBER-1": "+1 555 0100" + "input": { + "EMAIL-1": "alex@example.com" + }, + "output": { + "PHONE_NUMBER-1": "+1 555 0100" + } } } ``` diff --git a/docs/openapi/openapi.json b/docs/openapi/openapi.json index 31939af3ae3..142cd57c562 100644 --- a/docs/openapi/openapi.json +++ b/docs/openapi/openapi.json @@ -78668,10 +78668,21 @@ }, "redaction_mapping": { "type": "object", - "additionalProperties": { - "type": "string" + "properties": { + "input": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "output": { + "type": "object", + "additionalProperties": { + "type": "string" + } + } }, - "description": "Placeholder-to-original-value mapping for reversible redactions. Present only on log detail responses when the caller has `Logs:Reveal`." + "description": "Phase-scoped placeholder-to-original-value mappings for reversible redactions. Present only on log detail responses when the caller has `Logs:Reveal`." } } }, @@ -79319,4 +79330,4 @@ } } } -} \ No newline at end of file +} diff --git a/docs/openapi/schemas/management/logging.yaml b/docs/openapi/schemas/management/logging.yaml index d8494e6457e..585691a312c 100644 --- a/docs/openapi/schemas/management/logging.yaml +++ b/docs/openapi/schemas/management/logging.yaml @@ -57,9 +57,16 @@ LogEntry: type: string redaction_mapping: type: object - additionalProperties: - type: string - description: Placeholder-to-original-value mapping for reversible redactions. Present only on log detail responses when the caller has `Logs:Reveal`. + properties: + input: + type: object + additionalProperties: + type: string + output: + type: object + additionalProperties: + type: string + description: Phase-scoped placeholder-to-original-value mappings for reversible redactions. Present only on log detail responses when the caller has `Logs:Reveal`. created_at: type: string format: date-time diff --git a/framework/logstore/tables.go b/framework/logstore/tables.go index 95a4bc8d1d4..bb9a07da7e5 100644 --- a/framework/logstore/tables.go +++ b/framework/logstore/tables.go @@ -208,9 +208,9 @@ type Log struct { IsLargePayloadResponse bool `gorm:"default:false" json:"is_large_payload_response"` HasObject bool `gorm:"default:false" json:"-"` // True when payload is stored in object storage - RedactionData *schemas.RedactionData `gorm:"-" json:"-"` // Transient guardrail redaction data consumed by enterprise logstore wrappers - RedactionMapping string `gorm:"type:text" json:"-"` // Reversible redaction mapping (encrypted when an encryption key is set), written by enterprise logstore wrappers; deleted with the row - RevealRedactionMapping map[string]string `gorm:"-" json:"redaction_mapping,omitempty"` // Virtual field populated only on permitted log-detail reads + RedactionData *schemas.RedactionData `gorm:"-" json:"-"` // Transient guardrail redaction data consumed by enterprise logstore wrappers + RedactionMapping string `gorm:"type:text" json:"-"` // Reversible redaction mapping (encrypted when an encryption key is set), written by enterprise logstore wrappers; deleted with the row + RevealRedactionMapping *schemas.RedactionMapsByPhase `gorm:"-" json:"redaction_mapping,omitempty"` // Virtual field populated only on permitted log-detail reads // Cluster governance fields - attached by the logging plugin when running in a cluster // so that leaders can recover disconnected node usage from the logs table. diff --git a/framework/tracing/tracer.go b/framework/tracing/tracer.go index d427d76c7df..fe6b3ccaa5b 100644 --- a/framework/tracing/tracer.go +++ b/framework/tracing/tracer.go @@ -110,8 +110,8 @@ func (t *Tracer) SetTraceAttribute(traceID string, key string, value any) { t.store.SetTraceAttribute(traceID, key, value) } -// SetTraceRedactionReplacements stores connector-facing raw-to-placeholder replacements on a trace. -func (t *Tracer) SetTraceRedactionReplacements(traceID string, replacements map[string]string) { +// SetTraceRedactionReplacements stores phase-scoped connector-facing replacements on a trace. +func (t *Tracer) SetTraceRedactionReplacements(traceID string, phase schemas.RedactionPhase, replacements map[string]string) { if t == nil || t.store == nil || strings.TrimSpace(traceID) == "" || len(replacements) == 0 { return } @@ -119,7 +119,7 @@ func (t *Tracer) SetTraceRedactionReplacements(traceID string, replacements map[ if trace == nil { return } - trace.SetRedactionReplacements(replacements) + trace.SetRedactionReplacements(phase, replacements) } // CreateTrace creates a new trace with optional parent ID and returns the trace ID. diff --git a/framework/tracing/tracer_test.go b/framework/tracing/tracer_test.go index 06ebc34f6e5..65bc473e1c0 100644 --- a/framework/tracing/tracer_test.go +++ b/framework/tracing/tracer_test.go @@ -88,8 +88,11 @@ func TestTracer_CompleteAndFlushTraceRedactsContentBeforeInject(t *testing.T) { // Store replacements before output attributes are populated. This mirrors // streaming, where the final accumulated output lands near trace completion. - tracer.SetTraceRedactionReplacements(traceID, map[string]string{ - "alex@example.com": "[EMAIL-1]", + tracer.SetTraceRedactionReplacements(traceID, schemas.RedactionPhaseInput, map[string]string{ + "alex@example.com": "[EMAIL-INPUT]", + }) + tracer.SetTraceRedactionReplacements(traceID, schemas.RedactionPhaseOutput, map[string]string{ + "alex@example.com": "[EMAIL-OUTPUT]", }) ctx := context.WithValue(context.Background(), schemas.BifrostContextKeyTraceID, traceID) @@ -106,8 +109,11 @@ func TestTracer_CompleteAndFlushTraceRedactsContentBeforeInject(t *testing.T) { if strings.Contains(payload, "alex@example.com") { t.Fatalf("injected trace leaked raw content: %s", payload) } - if !strings.Contains(payload, "[EMAIL-1]") { - t.Fatalf("injected trace missing redacted placeholder: %s", payload) + if !strings.Contains(payload, "[EMAIL-INPUT]") { + t.Fatalf("injected trace missing input redacted placeholder: %s", payload) + } + if !strings.Contains(payload, "[EMAIL-OUTPUT]") { + t.Fatalf("injected trace missing output redacted placeholder: %s", payload) } if !strings.Contains(payload, "gpt-4o-mini") { t.Fatalf("injected trace should retain non-content attributes: %s", payload) @@ -125,7 +131,7 @@ func TestTracer_SetTraceRedactionReplacementsSurvivesLaterObservabilityPlugins(t defer tracer.Stop() traceID := tracer.CreateTrace("") - tracer.SetTraceRedactionReplacements(traceID, map[string]string{ + tracer.SetTraceRedactionReplacements(traceID, schemas.RedactionPhaseInput, map[string]string{ "alex@example.com": "[EMAIL-1]", }) diff --git a/plugins/logging/redaction_test.go b/plugins/logging/redaction_test.go index 95b1a61a9f5..bce4d4eb86a 100644 --- a/plugins/logging/redaction_test.go +++ b/plugins/logging/redaction_test.go @@ -15,43 +15,59 @@ import ( func TestAttachLogRedactionDataCopiesContextValue(t *testing.T) { ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) schemas.SetRedactionDataOnContext(ctx, schemas.RedactionData{ - ReversibleMappings: map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, - LiteralReplacements: map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"}, + LiteralReplacements: schemas.RedactionMapsByPhase{ + Input: map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"}, + Output: map[string]string{"rivera@example.com": "[EMAIL-2]"}, + }, + ReversibleMappings: schemas.RedactionMapsByPhase{ + Input: map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, + }, }) entry := &logstore.Log{} attachLogRedactionData(ctx, entry, true) require.NotNil(t, entry.RedactionData) - assert.Equal(t, map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, entry.RedactionData.ReversibleMappings) - assert.Equal(t, map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"}, entry.RedactionData.LiteralReplacements) + assert.Equal(t, map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, entry.RedactionData.ReversibleMappings.Input) + assert.Equal(t, map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"}, entry.RedactionData.LiteralReplacements.Input) + assert.Equal(t, map[string]string{"rivera@example.com": "[EMAIL-2]"}, entry.RedactionData.LiteralReplacements.Output) } // TestAttachLogRedactionDataClonesContextMaps verifies async log entries own their redaction maps. func TestAttachLogRedactionDataClonesContextMaps(t *testing.T) { ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) reversibleMappings := map[string]string{"EMAIL-1": "alex_rivera@gmail.com"} - literalReplacements := map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"} + inputLiteralReplacements := map[string]string{"alex_rivera@gmail.com": "[EMAIL-1]"} + outputLiteralReplacements := map[string]string{"rivera@example.com": "[EMAIL-2]"} schemas.SetRedactionDataOnContext(ctx, schemas.RedactionData{ - ReversibleMappings: reversibleMappings, - LiteralReplacements: literalReplacements, + LiteralReplacements: schemas.RedactionMapsByPhase{ + Input: inputLiteralReplacements, + Output: outputLiteralReplacements, + }, + ReversibleMappings: schemas.RedactionMapsByPhase{ + Input: reversibleMappings, + }, }) entry := &logstore.Log{} attachLogRedactionData(ctx, entry, true) reversibleMappings["EMAIL-1"] = "mutated@example.com" - literalReplacements["alex_rivera@gmail.com"] = "[MUTATED]" + inputLiteralReplacements["alex_rivera@gmail.com"] = "[MUTATED]" + outputLiteralReplacements["rivera@example.com"] = "[MUTATED]" require.NotNil(t, entry.RedactionData) - assert.Equal(t, "alex_rivera@gmail.com", entry.RedactionData.ReversibleMappings["EMAIL-1"]) - assert.Equal(t, "[EMAIL-1]", entry.RedactionData.LiteralReplacements["alex_rivera@gmail.com"]) + assert.Equal(t, "alex_rivera@gmail.com", entry.RedactionData.ReversibleMappings.Input["EMAIL-1"]) + assert.Equal(t, "[EMAIL-1]", entry.RedactionData.LiteralReplacements.Input["alex_rivera@gmail.com"]) + assert.Equal(t, "[EMAIL-2]", entry.RedactionData.LiteralReplacements.Output["rivera@example.com"]) } // TestAttachLogRedactionDataSkipsDisabledContentLogging verifies disabled content logging drops sensitive data. func TestAttachLogRedactionDataSkipsDisabledContentLogging(t *testing.T) { ctx := schemas.NewBifrostContext(context.Background(), time.Time{}) schemas.SetRedactionDataOnContext(ctx, schemas.RedactionData{ - ReversibleMappings: map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, + ReversibleMappings: schemas.RedactionMapsByPhase{ + Input: map[string]string{"EMAIL-1": "alex_rivera@gmail.com"}, + }, }) entry := &logstore.Log{} diff --git a/transports/bifrost-http/handlers/logging.go b/transports/bifrost-http/handlers/logging.go index af4febe3843..7e34397332f 100644 --- a/transports/bifrost-http/handlers/logging.go +++ b/transports/bifrost-http/handlers/logging.go @@ -206,10 +206,10 @@ type RedactedKeysManager interface { // LogRedactionMappingResolver optionally exposes decoded redaction mappings on log-detail responses. type LogRedactionMappingResolver interface { - // ResolveLogRedactionMapping returns a placeholder-to-original mapping when the caller may reveal it. + // ResolveLogRedactionMapping returns phase-scoped placeholder-to-original mappings when the caller may reveal them. // Implementations should return nil, nil when the caller is not authorized or no mapping is available. // Errors are treated as reveal-data failures only; the base log detail response is still served. - ResolveLogRedactionMapping(ctx *fasthttp.RequestCtx, log *logstore.Log) (map[string]string, error) + ResolveLogRedactionMapping(ctx *fasthttp.RequestCtx, log *logstore.Log) (*schemas.RedactionMapsByPhase, error) } // NewLoggingHandler creates a new logging handler instance @@ -620,7 +620,7 @@ func (h *LoggingHandler) getLogByID(ctx *fasthttp.RequestCtx) { mapping, err := h.logRedactionMappingResolver.ResolveLogRedactionMapping(ctx, log) if err != nil { logger.Error("failed to resolve redaction mapping for log %s: %v", id, err) - } else if len(mapping) > 0 { + } else if mapping != nil && mapping.HasReplacements() { log.RevealRedactionMapping = mapping } } diff --git a/ui/app/workspace/logs/sheets/logDetailView.tsx b/ui/app/workspace/logs/sheets/logDetailView.tsx index a7975f7a115..b5c4d74fad0 100644 --- a/ui/app/workspace/logs/sheets/logDetailView.tsx +++ b/ui/app/workspace/logs/sheets/logDetailView.tsx @@ -72,6 +72,18 @@ const getRealtimeTransportBadgeClass = (value: unknown): string => { } }; +const hasRedactionMappingEntries = (mapping?: LogEntry["redaction_mapping"]): boolean => + Boolean(mapping && (Object.keys(mapping.input ?? {}).length > 0 || Object.keys(mapping.output ?? {}).length > 0)); + +const applyRedactionMapping = (text: string | undefined, mapping?: Record): string => { + if (!text || !mapping) return text || ""; + let result = text; + for (const [key, value] of Object.entries(mapping)) { + result = result.replaceAll(`[${key}]`, value); + } + return result; +}; + const formatRealtimeSource = (value: unknown): string => { const source = String(value ?? "").trim(); switch (source.toLowerCase()) { @@ -576,9 +588,10 @@ export function LogDetailView({ }); const [showRevealedValues, setShowRevealedValues] = useState(false); const revealMapping = log?.redaction_mapping; - const revealAvailable = canReveal && Boolean(revealMapping && Object.keys(revealMapping).length > 0); + const revealAvailable = canReveal && hasRedactionMappingEntries(revealMapping); const revealEnabled = revealAvailable && showRevealedValues; - const activeRevealedMapping = revealEnabled ? revealMapping : undefined; + const activeInputRevealMapping = revealEnabled ? revealMapping?.input : undefined; + const activeOutputRevealMapping = revealEnabled ? revealMapping?.output : undefined; useEffect(() => { setShowRevealedValues(false); @@ -591,17 +604,6 @@ export function LogDetailView({ setShowRevealedValues(checked && revealAvailable); }; - // applyRedactionMapping replaces reversible placeholders like [EMAIL-1] with - // their original values when the reveal toggle is active. - const applyRedactionMapping = (text: string | undefined): string => { - if (!text || !activeRevealedMapping) return text || ""; - let result = text; - for (const [key, value] of Object.entries(activeRevealedMapping)) { - result = result.replaceAll(`[${key}]`, value); - } - return result; - }; - if (!log) return null; const selectedPromptDisplayName = resolvedSelectedPromptName ?? log.selected_prompt_name ?? ""; @@ -627,10 +629,10 @@ export function LogDetailView({ } const audioFormat = (log.params as any)?.audio?.format || (log.params as any)?.extra_params?.audio?.format || undefined; - const rawRequest = log.raw_request; - const rawResponse = log.raw_response; - const passthroughRequestBody = log.passthrough_request_body; - const passthroughResponseBody = log.passthrough_response_body; + const rawRequest = applyRedactionMapping(log.raw_request, activeInputRevealMapping); + const rawResponse = applyRedactionMapping(log.raw_response, activeOutputRevealMapping); + const passthroughRequestBody = applyRedactionMapping(log.passthrough_request_body, activeInputRevealMapping); + const passthroughResponseBody = applyRedactionMapping(log.passthrough_response_body, activeOutputRevealMapping); const videoOutput = log.video_generation_output || log.video_retrieve_output || log.video_download_output; const videoListOutput = log.video_list_output; const pluginLogCount = (() => { @@ -1822,8 +1824,8 @@ export function LogDetailView({ : log.input_history?.filter(Boolean) )?.flatMap((message, index) => { const role = ((message.role as string) || "user") as MessageRole; - const text = extractMessageText(message, activeRevealedMapping); - const reasoningText = extractChatReasoning(message, activeRevealedMapping); + const text = extractMessageText(message, activeInputRevealMapping); + const reasoningText = extractChatReasoning(message, activeInputRevealMapping); const showAll = visibleRoles.size === allRoles.length; const showMain = showAll || visibleRoles.has(role); const showReasoning = !!reasoningText && (showAll || visibleRoles.has("reasoning")); @@ -1921,12 +1923,12 @@ export function LogDetailView({ {log.output_message && !log.error_details?.error.message && (() => { - const reasoningText = extractChatReasoning(log.output_message, activeRevealedMapping); + const reasoningText = extractChatReasoning(log.output_message, activeOutputRevealMapping); const showReasoning = !!reasoningText && (visibleRoles.size === allRoles.length || visibleRoles.has("reasoning")); const showAssistant = visibleRoles.has("assistant"); if (!showReasoning && !showAssistant) return null; - const text = extractMessageText(log.output_message, activeRevealedMapping); - const refusalText = applyRedactionMapping(log.output_message.refusal); + const text = extractMessageText(log.output_message, activeOutputRevealMapping); + const refusalText = applyRedactionMapping(log.output_message.refusal, activeOutputRevealMapping); const isStopReasonRefusal = log.stop_reason === "refusal" || log.stop_reason === "content_filter" || log.stop_reason === "safety"; const showRefusal = refusalText || (!text && isStopReasonRefusal); @@ -2016,21 +2018,24 @@ export function LogDetailView({ const rawOutput = log.status !== "processing" && !log.error_details?.error.message ? (log.responses_output ?? []) : []; const outputMsgs = visibleRoles.size < allRoles.length ? rawOutput.filter((m) => visibleRoles.has(getResponsesRole(m))) : rawOutput; - const all: ResponsesMessage[] = coalesceResponsesMessages([...inputMsgs, ...outputMsgs]); + const all: Array<{ msg: ResponsesMessage; mapping?: Record }> = [ + ...coalesceResponsesMessages(inputMsgs).map((msg) => ({ msg, mapping: activeInputRevealMapping })), + ...coalesceResponsesMessages(outputMsgs).map((msg) => ({ msg, mapping: activeOutputRevealMapping })), + ]; if (all.length === 0) return null; return (
- {all.map((msg, index) => { + {all.map(({ msg, mapping }, index) => { const role = getResponsesRole(msg); const isLast = index === all.length - 1; - const reasoningParts = role === "reasoning" ? extractReasoningParts(msg, activeRevealedMapping) : null; + const reasoningParts = role === "reasoning" ? extractReasoningParts(msg, mapping) : null; const reasoningHasAny = !!reasoningParts && (reasoningParts.summaries.length > 0 || !!reasoningParts.encrypted || !!reasoningParts.contentText || reasoningParts.signatures.length > 0); - const text = role === "reasoning" ? "" : extractResponsesText(msg, activeRevealedMapping); + const text = role === "reasoning" ? "" : extractResponsesText(msg, mapping); const lineCount = text ? text.split("\n").length : 0; const approxTokens = text ? Math.max(1, Math.round(text.length / 4)) : 0; let meta: string | undefined; diff --git a/ui/lib/types/logs.ts b/ui/lib/types/logs.ts index b698e511434..e1a8520c4f3 100644 --- a/ui/lib/types/logs.ts +++ b/ui/lib/types/logs.ts @@ -581,7 +581,10 @@ export interface LogEntry { passthrough_request_body?: string; // Raw passthrough request body (UTF-8) passthrough_response_body?: string; // Raw passthrough response body (UTF-8) metadata?: Record; // JSON metadata (e.g., isAsyncRequest) - redaction_mapping?: Record; // Placeholder-to-original mapping, present only when caller has Logs:Reveal + redaction_mapping?: { + input?: Record; + output?: Record; + }; // Phase-scoped placeholder-to-original mappings, present only when caller has Logs:Reveal } export interface LogFilters {