diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 610078d225ea..2c1c772c1d81 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -208,6 +208,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936] - Add field redaction package. {pull}40997[40997] +- Add support for marked redaction to x-pack/filebeat/input/internal/private {pull}41212[41212] ==== Deprecated diff --git a/x-pack/filebeat/input/internal/private/private.go b/x-pack/filebeat/input/internal/private/private.go index e47b6521e477..c0b2e311fded 100644 --- a/x-pack/filebeat/input/internal/private/private.go +++ b/x-pack/filebeat/input/internal/private/private.go @@ -35,7 +35,11 @@ var privateKey = reflect.ValueOf("private") // `private:""`, the fields with the tag will be marked as private. Otherwise // the comma-separated list of names with be used. The list may refer to its // own field. -func Redact[T any](val T, tag string, global []string) (redacted T, err error) { +func Redact[T any](val T, tag string, global []string, replace ...Replacer) (redacted T, err error) { + reps, err := compileReplacers(replace) + if err != nil { + return redacted, err + } defer func() { switch r := recover().(type) { case nil: @@ -54,13 +58,65 @@ func Redact[T any](val T, tag string, global []string) (redacted T, err error) { rv := reflect.ValueOf(val) switch rv.Kind() { case reflect.Map, reflect.Pointer, reflect.Struct: - return redact(rv, tag, slices.Clone(global), 0, make(map[any]int)).Interface().(T), nil + return redact(rv, reps, tag, slices.Clone(global), 0, make(map[any]int)).Interface().(T), nil default: return val, nil } } -func redact(v reflect.Value, tag string, global []string, depth int, seen map[any]int) reflect.Value { +// Replacer is a function that will return a redaction replacement +// for the provided type. It must be a func(T) T. +type Replacer any + +// NewStringReplacer returns a string Replacer that returns s. +func NewStringReplacer(s string) Replacer { + return func(string) string { + return s + } +} + +// NewBytesReplacer returns a []byte Replacer that returns the bytes +// representation of s. +func NewBytesReplacer(s string) Replacer { + return func([]byte) []byte { + return []byte(s) + } +} + +type replacers map[reflect.Type]func(reflect.Value) reflect.Value + +func compileReplacers(replace []Replacer) (replacers, error) { + reps := make(replacers) + for _, r := range replace { + rv := reflect.ValueOf(r) + rt := rv.Type() + if rt.Kind() != reflect.Func { + return nil, fmt.Errorf("replacer is not a function: %T", r) + } + if n := rt.NumIn(); n != 1 { + return nil, fmt.Errorf("incorrect number of arguments for replacer: %d != 1", n) + } + if n := rt.NumOut(); n != 1 { + return nil, fmt.Errorf("incorrect number of return values from replacer: %d != 1", n) + } + in, out := rt.In(0), rt.Out(0) + if in != out { + return nil, fmt.Errorf("replacer does not preserve type: fn(%s) %s", in, out) + } + if _, exists := reps[in]; exists { + return nil, fmt.Errorf("multiple replacers for %s", in) + } + reps[in] = func(v reflect.Value) reflect.Value { + return rv.Call([]reflect.Value{v})[0] + } + } + if len(reps) == 0 { + reps = nil + } + return reps, nil +} + +func redact(v reflect.Value, reps replacers, tag string, global []string, depth int, seen map[any]int) reflect.Value { switch v.Kind() { case reflect.Pointer: if v.IsNil() { @@ -74,19 +130,19 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an seen[ident] = depth defer delete(seen, ident) } - return redact(v.Elem(), tag, global, depth+1, seen).Addr() + return redact(v.Elem(), reps, tag, global, depth+1, seen).Addr() case reflect.Interface: if v.IsNil() { return v } - return redact(v.Elem(), tag, global, depth+1, seen) + return redact(v.Elem(), reps, tag, global, depth+1, seen) case reflect.Array: if v.Len() == 0 { return v } r := reflect.New(v.Type()).Elem() for i := 0; i < v.Len(); i++ { - r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen)) + r.Index(i).Set(redact(v.Index(i), reps, tag, global, depth+1, seen)) } return r case reflect.Slice: @@ -109,7 +165,7 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an } r := reflect.MakeSlice(v.Type(), v.Len(), v.Cap()) for i := 0; i < v.Len(); i++ { - r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen)) + r.Index(i).Set(redact(v.Index(i), reps, tag, global, depth+1, seen)) } return r case reflect.Map: @@ -145,9 +201,13 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an for it.Next() { name := it.Key().String() if slices.Contains(private, name) { + v := replaceNestedWithin(it.Value(), reps) + if v.IsValid() { + r.SetMapIndex(it.Key(), v) + } continue } - r.SetMapIndex(it.Key(), redact(it.Value(), tag, nextPath(name, global), depth+1, seen)) + r.SetMapIndex(it.Key(), redact(it.Value(), reps, tag, nextPath(name, global), depth+1, seen)) } return r case reflect.Struct: @@ -219,10 +279,14 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an continue } if slices.Contains(private, names[i]) { + v := replaceNestedWithin(f, reps) + if v.IsValid() { + r.Field(i).Set(v) + } continue } if r.Field(i).CanSet() { - r.Field(i).Set(redact(f, tag, nextPath(names[i], global), depth+1, seen)) + r.Field(i).Set(redact(f, reps, tag, nextPath(names[i], global), depth+1, seen)) } } return r @@ -230,6 +294,67 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an return v } +// replaceNestedWithin replaces deeply nested values in pointer, interface and +// array/slice chains. If a replacement is not made an invalid reflect.Value +// is returned. If elements are not replaced by a replacer, it is set to the +// zero value for the type. +func replaceNestedWithin(v reflect.Value, reps replacers) reflect.Value { + if len(reps) == 0 || !v.IsValid() { + // No replacer, or an invalid value, so fall back to removal. + return reflect.Value{} + } + if rep, ok := reps[v.Type()]; ok { + return rep(v) + } + switch v.Kind() { + case reflect.Pointer: + r := replaceNestedWithin(v.Elem(), reps) + if !r.IsValid() { + return r + } + return r.Addr() + case reflect.Interface: + r := replaceNestedWithin(v.Elem(), reps) + if !r.IsValid() { + return r + } + i := reflect.New(v.Type()).Elem() + i.Set(r) + return i + case reflect.Array: + a := reflect.New(v.Type()).Elem() + wasSet := false + for i := 0; i < v.Len(); i++ { + r := replaceNestedWithin(v.Index(i), reps) + if r.IsValid() { + wasSet = true + a.Index(i).Set(r) + } + } + if !wasSet { + return reflect.Value{} + } + return a + case reflect.Slice: + s := reflect.MakeSlice(v.Type(), v.Len(), v.Cap()) + wasSet := false + for i := 0; i < v.Len(); i++ { + r := replaceNestedWithin(v.Index(i), reps) + if r.IsValid() { + wasSet = true + s.Index(i).Set(r) + } + } + if !wasSet { + return reflect.Value{} + } + return s + default: + // Could not catch, fall back to removal. + return reflect.Value{} + } +} + func nextStep(global []string) (private []string) { if len(global) == 0 { return nil diff --git a/x-pack/filebeat/input/internal/private/private_test.go b/x-pack/filebeat/input/internal/private/private_test.go index 774e35f3d532..aa813ada5d1b 100644 --- a/x-pack/filebeat/input/internal/private/private_test.go +++ b/x-pack/filebeat/input/internal/private/private_test.go @@ -7,20 +7,23 @@ package private import ( "bytes" "encoding/json" + "errors" "net/url" "reflect" + "strings" "testing" "github.com/google/go-cmp/cmp" ) type redactTest struct { - name string - in any - tag string - global []string - want any - wantErr error + name string + in any + tag string + global []string + replacers []Replacer + want any + wantErr error } var redactTests = []redactTest{ @@ -36,6 +39,34 @@ var redactTests = []redactTest{ "not_secret": "2", }, }, + { + name: "map_string_replacer", + in: map[string]any{ + "private": "secret", + "secret": "this is a secret", + "not_secret": "this is not", + }, + replacers: []Replacer{NewStringReplacer("REDACTED")}, + want: map[string]any{ + "private": "secret", + "secret": "REDACTED", + "not_secret": "this is not", + }, + }, + { + name: "map_string_custom_replacer", + in: map[string]any{ + "private": "secret", + "secret": "this is a secret", + "not_secret": "this is not", + }, + replacers: []Replacer{func(s string) string { return strings.Repeat("*", len(s)) }}, + want: map[string]any{ + "private": "secret", + "secret": "****************", // Same length as original. + "not_secret": "this is not", + }, + }, { name: "map_string_inner", in: map[string]any{ @@ -80,6 +111,78 @@ var redactTests = []redactTest{ }, }}, }, + { + name: "map_string_inner_next_inner_global_slices", + in: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "secret": []string{"1"}, + "not_secret": []string{"2"}, + }, + }}, + global: []string{"inner.next_inner.secret"}, + want: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "not_secret": []string{"2"}, + }, + }}, + }, + { + name: "map_string_inner_next_inner_global_nested_slices", + in: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "secret": [][]string{{"1"}}, + "not_secret": [][]string{{"2"}}, + }, + }}, + global: []string{"inner.next_inner.secret"}, + want: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "not_secret": [][]string{{"2"}}, + }, + }}, + }, + { + name: "map_string_inner_next_inner_global_slices_replacer", + in: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "secret": []string{"1"}, + "not_secret": []string{"2"}, + }, + }}, + replacers: []Replacer{NewStringReplacer("REDACTED")}, + global: []string{"inner.next_inner.secret"}, + want: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "not_secret": []string{"2"}, + "secret": []string{"REDACTED"}, + }, + }}, + }, + { + name: "map_string_inner_next_inner_global_nested_slices_replacer", + in: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "secret": [][]string{{"1"}}, + "not_secret": [][]string{{"2"}}, + }, + }}, + replacers: []Replacer{NewStringReplacer("REDACTED")}, + global: []string{"inner.next_inner.secret"}, + want: map[string]any{ + "inner": map[string]any{ + "next_inner": map[string]any{ + "secret": [][]string{{"REDACTED"}}, + "not_secret": [][]string{{"2"}}, + }, + }}, + }, { name: "map_string_inner_next_inner_params_global", in: map[string]any{ @@ -193,6 +296,49 @@ var redactTests = []redactTest{ }, }}, }, + { + name: "map_string_inner_next_inner_params_global_internal_slice_precise_replacer", + in: map[string]any{ + "inner": map[string]any{ + "next_inner": []map[string]any{ + { + "headers": url.Values{ + "secret": []string{"1"}, + "not_secret": []string{"2"}, + }, + "not_secret": "2", + }, + { + "headers": url.Values{ + "secret": []string{"3"}, + "not_secret": []string{"4"}, + }, + "not_secret": "4", + }, + }, + }}, + global: []string{"inner.next_inner.headers.secret"}, + replacers: []Replacer{NewStringReplacer("REDACTED")}, + want: map[string]any{ + "inner": map[string]any{ + "next_inner": []map[string]any{ + { + "headers": url.Values{ + "not_secret": []string{"2"}, + "secret": []string{"REDACTED"}, + }, + "not_secret": "2", + }, + { + "headers": url.Values{ + "not_secret": []string{"4"}, + "secret": []string{"REDACTED"}, + }, + "not_secret": "4", + }, + }, + }}, + }, { name: "map_slice", in: map[string]any{ @@ -239,6 +385,50 @@ var redactTests = []redactTest{ }, } }(), + func() redactTest { + type s struct { + Private string + Secret string + NotSecret string + } + return redactTest{ + name: "struct_string_replacer", + in: s{ + Private: "Secret", + Secret: "this is a secret", + NotSecret: "this is not", + }, + replacers: []Replacer{NewStringReplacer("REDACTED")}, + tag: "", + want: s{ + Private: "Secret", + Secret: "REDACTED", + NotSecret: "this is not", + }, + } + }(), + func() redactTest { + type s struct { + Private string + Secret string + NotSecret string + } + return redactTest{ + name: "struct_string_replacer", + in: s{ + Private: "Secret", + Secret: "this is a secret", + NotSecret: "this is not", + }, + replacers: []Replacer{func(s string) string { return strings.Repeat("*", len(s)) }}, + tag: "", + want: s{ + Private: "Secret", + Secret: "****************", + NotSecret: "this is not", + }, + } + }(), func() redactTest { type s struct { Private []string @@ -399,6 +589,37 @@ var redactTests = []redactTest{ wantErr: cycle{reflect.TypeOf(&s{})}, } }(), + { + name: "invalid_replacer_wrong_type", + in: struct{}{}, + replacers: []Replacer{func(s string) int { return len(s) }}, + want: struct{}{}, + wantErr: errors.New("replacer does not preserve type: fn(string) int"), + }, + { + name: "invalid_replacer_wrong_argnum", + in: struct{}{}, + replacers: []Replacer{func(a, b string) string { return a + b }}, + want: struct{}{}, + wantErr: errors.New("incorrect number of arguments for replacer: 2 != 1"), + }, + { + name: "invalid_replacer_wrong_retnum", + in: struct{}{}, + replacers: []Replacer{func(s string) (a, b string) { return s, s }}, + want: struct{}{}, + wantErr: errors.New("incorrect number of return values from replacer: 2 != 1"), + }, + { + name: "invalid_replacer_collision", + in: struct{}{}, + replacers: []Replacer{ + func(s string) string { return s }, + func(s string) string { return s }, + }, + want: struct{}{}, + wantErr: errors.New("multiple replacers for string"), + }, } func TestRedact(t *testing.T) { @@ -415,10 +636,13 @@ func TestRedact(t *testing.T) { t.Fatalf("failed to get before state: %v", err) } } - got, err := Redact(test.in, test.tag, test.global) - if err != test.wantErr { + got, err := Redact(test.in, test.tag, test.global, test.replacers...) + if !sameError(err, test.wantErr) { t.Fatalf("unexpected error from Redact: %v", err) } + if err != nil { + return + } if !isCycle { after, err := json.Marshal(test.in) if err != nil { @@ -434,3 +658,14 @@ func TestRedact(t *testing.T) { }) } } + +func sameError(a, b error) bool { + switch { + case a == nil && b == nil: + return true + case a == nil, b == nil: + return false + default: + return a.Error() == b.Error() + } +}