Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 74 additions & 23 deletions core/schemas/redaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,57 @@ import (
"strings"
)

// RedactionPhase identifies which request lifecycle phase produced a redaction finding.
type RedactionPhase string

const (
// RedactionPhaseInput marks redaction findings discovered while inspecting request-side content.
RedactionPhaseInput RedactionPhase = "input"
// RedactionPhaseOutput marks redaction findings discovered while inspecting response-side content.
RedactionPhaseOutput RedactionPhase = "output"
)

// RedactionData carries request-scoped redaction data from guardrails to log and trace sinks.
type RedactionData struct {
ReversibleMappings map[string]string `json:"reversible_mappings,omitempty"`
LiteralReplacements map[string]string `json:"literal_replacements,omitempty"`
LiteralReplacements RedactionMapsByPhase `json:"literal_replacements,omitempty"`
ReversibleMappings RedactionMapsByPhase `json:"reversible_mappings,omitempty"`
}

// RedactionMapsByPhase stores replacement maps separately for request and response content.
type RedactionMapsByPhase struct {
Input map[string]string `json:"input,omitempty"`
Output map[string]string `json:"output,omitempty"`
}

// Clone returns an owned copy of the phase-scoped maps.
func (m RedactionMapsByPhase) Clone() RedactionMapsByPhase {
return RedactionMapsByPhase{
Input: maps.Clone(m.Input),
Output: maps.Clone(m.Output),
}
}

// HasReplacements reports whether either phase has replacement entries.
func (m RedactionMapsByPhase) HasReplacements() bool {
return len(m.Input) > 0 || len(m.Output) > 0
}

// MergePhase merges replacements into one phase, copying entries so callers cannot mutate stored state.
func (m *RedactionMapsByPhase) MergePhase(phase RedactionPhase, replacements map[string]string) {
if m == nil || len(replacements) == 0 {
return
}
switch phase {
case RedactionPhaseInput:
m.Input = mergeRedactionStringMaps(m.Input, replacements)
case RedactionPhaseOutput:
m.Output = mergeRedactionStringMaps(m.Output, replacements)
}
}

// MergedForMixedFields returns both phase maps for fields that can contain input and output.
func (m RedactionMapsByPhase) MergedForMixedFields() map[string]string {
return mergeRedactionStringMaps(m.Input, m.Output)
}

Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
// Clone returns an owned snapshot of the redaction data maps.
Expand All @@ -21,26 +68,31 @@ type RedactionData struct {
// contain immutable strings.
func (d RedactionData) Clone() RedactionData {
return RedactionData{
ReversibleMappings: maps.Clone(d.ReversibleMappings),
LiteralReplacements: maps.Clone(d.LiteralReplacements),
LiteralReplacements: d.LiteralReplacements.Clone(),
ReversibleMappings: d.ReversibleMappings.Clone(),
}
}

// HasReplacements reports whether any reversible or literal redaction data is present.
func (d RedactionData) HasReplacements() bool {
return d.LiteralReplacements.HasReplacements() || d.ReversibleMappings.HasReplacements()
}

// RedactionDataFromContext returns the redaction data stored on ctx.
func RedactionDataFromContext(ctx *BifrostContext) (RedactionData, bool) {
if ctx == nil {
return RedactionData{}, false
}
data, ok := ctx.Value(BifrostContextKeyRedactionData).(RedactionData)
if !ok || (len(data.ReversibleMappings) == 0 && len(data.LiteralReplacements) == 0) {
if !ok || !data.HasReplacements() {
return RedactionData{}, false
}
return data, true
}

// SetRedactionDataOnContext stores non-empty redaction data on ctx.
func SetRedactionDataOnContext(ctx *BifrostContext, data RedactionData) bool {
if ctx == nil || (len(data.ReversibleMappings) == 0 && len(data.LiteralReplacements) == 0) {
if ctx == nil || !data.HasReplacements() {
return false
}
ctx.SetValue(BifrostContextKeyRedactionData, data)
Expand All @@ -49,23 +101,7 @@ func SetRedactionDataOnContext(ctx *BifrostContext, data RedactionData) bool {

// IsContentAttribute reports whether a span attribute may carry user or model content.
func IsContentAttribute(key string) bool {
switch key {
case AttrInputMessages, AttrOutputMessages,
AttrInputText, AttrInputSpeech,
AttrInputEmbedding,
AttrPrompt, AttrInstructions,
AttrRespReasoningText:
return true
case AttrTools, AttrRespTools,
AttrToolName, AttrToolCallID,
AttrToolCallArguments, AttrToolCallResult,
AttrToolType,
AttrToolChoiceType, AttrToolChoiceName,
AttrRespToolChoiceType, AttrRespToolChoiceName:
return true
default:
return false
}
return traceContentAttributeScopeForKey(key) != traceContentAttributeScopeNone
}

// ApplyLiteralReplacements performs deterministic best-effort string redaction.
Expand Down Expand Up @@ -130,3 +166,18 @@ func RedactAttributeValue(value any, replacements map[string]string) any {
return value
}
}

// mergeRedactionStringMaps returns a copied map containing both map values, with next taking precedence.
func mergeRedactionStringMaps(current map[string]string, next map[string]string) map[string]string {
if len(current) == 0 && len(next) == 0 {
return nil
}
merged := maps.Clone(current)
if merged == nil {
merged = make(map[string]string, len(next))
}
for key, value := range next {
merged[key] = value
}
return merged
}
83 changes: 54 additions & 29 deletions core/schemas/redaction_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,14 @@ import (
func TestRedactionDataContextRoundTrip(t *testing.T) {
ctx := NewBifrostContext(context.Background(), NoDeadline)
data := RedactionData{
ReversibleMappings: map[string]string{"EMAIL-1": "alex@example.com"},
LiteralReplacements: map[string]string{"alex@example.com": "[EMAIL-1]"},
LiteralReplacements: RedactionMapsByPhase{
Input: map[string]string{"alex@example.com": "[EMAIL-1]"},
Output: map[string]string{"rivera@example.com": "[EMAIL-2]"},
},
ReversibleMappings: RedactionMapsByPhase{
Input: map[string]string{"EMAIL-1": "alex@example.com"},
Output: map[string]string{"EMAIL-2": "rivera@example.com"},
},
}

require.True(t, SetRedactionDataOnContext(ctx, data))
Expand All @@ -28,16 +34,26 @@ func TestRedactionDataContextRoundTrip(t *testing.T) {
// TestRedactionDataCloneCopiesMaps verifies clones do not share mutable map storage.
func TestRedactionDataCloneCopiesMaps(t *testing.T) {
data := RedactionData{
ReversibleMappings: map[string]string{"EMAIL-1": "alex@example.com"},
LiteralReplacements: map[string]string{"alex@example.com": "[EMAIL-1]"},
LiteralReplacements: RedactionMapsByPhase{
Input: map[string]string{"alex@example.com": "[EMAIL-1]"},
Output: map[string]string{"rivera@example.com": "[EMAIL-2]"},
},
ReversibleMappings: RedactionMapsByPhase{
Input: map[string]string{"EMAIL-1": "alex@example.com"},
Output: map[string]string{"EMAIL-2": "rivera@example.com"},
},
}

clone := data.Clone()
data.ReversibleMappings["EMAIL-1"] = "mutated@example.com"
data.LiteralReplacements["alex@example.com"] = "[MUTATED]"

assert.Equal(t, "alex@example.com", clone.ReversibleMappings["EMAIL-1"])
assert.Equal(t, "[EMAIL-1]", clone.LiteralReplacements["alex@example.com"])
data.ReversibleMappings.Input["EMAIL-1"] = "mutated@example.com"
data.ReversibleMappings.Output["EMAIL-2"] = "mutated@example.com"
data.LiteralReplacements.Input["alex@example.com"] = "[MUTATED]"
data.LiteralReplacements.Output["rivera@example.com"] = "[MUTATED]"

assert.Equal(t, "alex@example.com", clone.ReversibleMappings.Input["EMAIL-1"])
assert.Equal(t, "rivera@example.com", clone.ReversibleMappings.Output["EMAIL-2"])
assert.Equal(t, "[EMAIL-1]", clone.LiteralReplacements.Input["alex@example.com"])
assert.Equal(t, "[EMAIL-2]", clone.LiteralReplacements.Output["rivera@example.com"])
}

// TestRedactionDataFromContextRejectsSerializedValues verifies the handoff remains typed.
Expand Down Expand Up @@ -77,57 +93,62 @@ func TestIsContentAttribute(t *testing.T) {
assert.False(t, IsContentAttribute(TraceAttrSessionID))
}

// TestTraceApplyRedactionReplacementsRedactsContentAttributes verifies trace redaction touches all spans.
// TestTraceApplyRedactionReplacementsRedactsContentAttributes verifies trace redaction honors attribute phase.
func TestTraceApplyRedactionReplacementsRedactsContentAttributes(t *testing.T) {
trace := &Trace{}
root := &Span{}
child := &Span{}
trace.RootSpan = root
trace.Spans = []*Span{root, child}

root.SetAttribute(AttrInputMessages, `{"content":"email alex@example.com"}`)
root.SetAttribute(AttrInputMessages, `{"content":"email alex@example.com and bob@example.com"}`)
root.SetAttribute(AttrRequestModel, "alex@example.com")
child.SetAttribute(AttrOutputMessages, []string{"reply to alex@example.com"})
child.SetAttribute(AttrOutputMessages, []string{"reply to alex@example.com and bob@example.com"})
child.SetAttribute(AttrToolCallArguments, map[string]any{
"customer": map[string]any{
"email": "alex@example.com",
"tags": []any{"safe", "alex@example.com"},
"owner": "bob@example.com",
"tags": []any{"safe", "alex@example.com", "bob@example.com"},
},
"metadata": map[string]string{
"owner": "alex@example.com",
"owner": "bob@example.com",
},
"literal_key_alex@example.com": "key should redact too",
"count": 42,
})
child.AddEvent(SpanEvent{
Name: "llm.message",
Attributes: map[string]any{
AttrInputMessages: `{"content":"event alex@example.com"}`,
AttrRequestModel: "alex@example.com",
AttrInputMessages: `{"content":"event alex@example.com and bob@example.com"}`,
AttrOutputMessages: `{"content":"event alex@example.com and bob@example.com"}`,
AttrRequestModel: "alex@example.com",
},
})

trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"bob@example.com": "[EMAIL-2]"})
trace.ApplyRedactionReplacements()

assert.Equal(t, `{"content":"email [EMAIL-1]"}`, root.Attributes[AttrInputMessages])
assert.Equal(t, `{"content":"email [EMAIL-1] and bob@example.com"}`, root.Attributes[AttrInputMessages])
assert.Equal(t, "alex@example.com", root.Attributes[AttrRequestModel])
assert.Equal(t, []string{"reply to [EMAIL-1]"}, child.Attributes[AttrOutputMessages])
assert.Equal(t, []string{"reply to alex@example.com and [EMAIL-2]"}, child.Attributes[AttrOutputMessages])
assert.Equal(t, map[string]any{
"customer": map[string]any{
"email": "[EMAIL-1]",
"tags": []any{"safe", "[EMAIL-1]"},
"owner": "[EMAIL-2]",
"tags": []any{"safe", "[EMAIL-1]", "[EMAIL-2]"},
},
"metadata": map[string]string{
"owner": "[EMAIL-1]",
"owner": "[EMAIL-2]",
},
"literal_key_[EMAIL-1]": "key should redact too",
"count": 42,
}, child.Attributes[AttrToolCallArguments])
require.Len(t, child.Events, 1)
assert.Equal(t, `{"content":"event [EMAIL-1]"}`, child.Events[0].Attributes[AttrInputMessages])
assert.Equal(t, `{"content":"event [EMAIL-1] and bob@example.com"}`, child.Events[0].Attributes[AttrInputMessages])
assert.Equal(t, `{"content":"event alex@example.com and [EMAIL-2]"}`, child.Events[0].Attributes[AttrOutputMessages])
assert.Equal(t, "alex@example.com", child.Events[0].Attributes[AttrRequestModel])
assert.Nil(t, trace.redactionReplacements)
assert.False(t, trace.redactionReplacements.HasReplacements())
}

// TestTraceSetRedactionReplacementsMergesCalls verifies input/output replacement windows accumulate.
Expand All @@ -141,35 +162,39 @@ func TestTraceSetRedactionReplacementsMergesCalls(t *testing.T) {
root.SetAttribute(AttrInputMessages, `{"content":"email input@example.com"}`)
child.SetAttribute(AttrOutputMessages, `{"content":"reply output@example.com"}`)

trace.SetRedactionReplacements(map[string]string{"input@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(map[string]string{"output@example.com": "[EMAIL-2]"})
trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"input@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"output@example.com": "[EMAIL-2]"})
trace.ApplyRedactionReplacements()

assert.Equal(t, `{"content":"email [EMAIL-1]"}`, root.Attributes[AttrInputMessages])
assert.Equal(t, `{"content":"reply [EMAIL-2]"}`, child.Attributes[AttrOutputMessages])
assert.Nil(t, trace.redactionReplacements)
assert.False(t, trace.redactionReplacements.HasReplacements())
}

// TestTraceRedactionReplacementsDoNotSerialize verifies connector-facing replacements stay internal.
func TestTraceRedactionReplacementsDoNotSerialize(t *testing.T) {
trace := &Trace{TraceID: "trace-1"}
trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"rivera@example.com": "[EMAIL-2]"})

serialized, err := sonic.MarshalString(trace)
require.NoError(t, err)

assert.NotContains(t, serialized, "alex@example.com")
assert.NotContains(t, serialized, "[EMAIL-1]")
assert.NotContains(t, serialized, "rivera@example.com")
assert.NotContains(t, serialized, "[EMAIL-2]")
assert.False(t, strings.Contains(serialized, "redactionReplacements"))
assert.False(t, strings.Contains(serialized, "RedactionReplacements"))
}

// TestTraceResetClearsRedactionReplacements verifies pooled traces cannot retain request redaction data.
func TestTraceResetClearsRedactionReplacements(t *testing.T) {
trace := &Trace{}
trace.SetRedactionReplacements(map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseInput, map[string]string{"alex@example.com": "[EMAIL-1]"})
trace.SetRedactionReplacements(RedactionPhaseOutput, map[string]string{"rivera@example.com": "[EMAIL-2]"})

trace.Reset()

assert.Nil(t, trace.redactionReplacements)
assert.False(t, trace.redactionReplacements.HasReplacements())
}
Loading
Loading