[8.x](backport #41212) x-pack/filebeat/input/internal/private: add support for marked redaction

CHANGELOG-developer.next.asciidoc

@@ -208,6 +208,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Simplified GCS input state checkpoint calculation logic. {issue}40878[40878] {pull}40937[40937] 
 - Simplified Azure Blob Storage input state checkpoint calculation logic. {issue}40674[40674] {pull}40936[40936]
 - Add field redaction package. {pull}40997[40997]
+- Add support for marked redaction to x-pack/filebeat/input/internal/private {pull}41212[41212]
 
 ==== Deprecated
 

x-pack/filebeat/input/internal/private/private.go

@@ -35,7 +35,11 @@ var privateKey = reflect.ValueOf("private")
 // `private:""`, the fields with the tag will be marked as private. Otherwise
 // the comma-separated list of names with be used. The list may refer to its
 // own field.
-func Redact[T any](val T, tag string, global []string) (redacted T, err error) {
+func Redact[T any](val T, tag string, global []string, replace ...Replacer) (redacted T, err error) {
+	reps, err := compileReplacers(replace)
+	if err != nil {
+		return redacted, err
+	}
 	defer func() {
 		switch r := recover().(type) {
 		case nil:
@@ -54,13 +58,65 @@ func Redact[T any](val T, tag string, global []string) (redacted T, err error) {
 	rv := reflect.ValueOf(val)
 	switch rv.Kind() {
 	case reflect.Map, reflect.Pointer, reflect.Struct:
-		return redact(rv, tag, slices.Clone(global), 0, make(map[any]int)).Interface().(T), nil
+		return redact(rv, reps, tag, slices.Clone(global), 0, make(map[any]int)).Interface().(T), nil
 	default:
 		return val, nil
 	}
 }
 
-func redact(v reflect.Value, tag string, global []string, depth int, seen map[any]int) reflect.Value {
+// Replacer is a function that will return a redaction replacement
+// for the provided type. It must be a func(T) T.
+type Replacer any
+
+// NewStringReplacer returns a string Replacer that returns s.
+func NewStringReplacer(s string) Replacer {
+	return func(string) string {
+		return s
+	}
+}
+
+// NewBytesReplacer returns a []byte Replacer that returns the bytes
+// representation of s.
+func NewBytesReplacer(s string) Replacer {
+	return func([]byte) []byte {
+		return []byte(s)
+	}
+}
+
+type replacers map[reflect.Type]func(reflect.Value) reflect.Value
+
+func compileReplacers(replace []Replacer) (replacers, error) {
+	reps := make(replacers)
+	for _, r := range replace {
+		rv := reflect.ValueOf(r)
+		rt := rv.Type()
+		if rt.Kind() != reflect.Func {
+			return nil, fmt.Errorf("replacer is not a function: %T", r)
+		}
+		if n := rt.NumIn(); n != 1 {
+			return nil, fmt.Errorf("incorrect number of arguments for replacer: %d != 1", n)
+		}
+		if n := rt.NumOut(); n != 1 {
+			return nil, fmt.Errorf("incorrect number of return values from replacer: %d != 1", n)
+		}
+		in, out := rt.In(0), rt.Out(0)
+		if in != out {
+			return nil, fmt.Errorf("replacer does not preserve type: fn(%s) %s", in, out)
+		}
+		if _, exists := reps[in]; exists {
+			return nil, fmt.Errorf("multiple replacers for %s", in)
+		}
+		reps[in] = func(v reflect.Value) reflect.Value {
+			return rv.Call([]reflect.Value{v})[0]
+		}
+	}
+	if len(reps) == 0 {
+		reps = nil
+	}
+	return reps, nil
+}
+
+func redact(v reflect.Value, reps replacers, tag string, global []string, depth int, seen map[any]int) reflect.Value {
 	switch v.Kind() {
 	case reflect.Pointer:
 		if v.IsNil() {
@@ -74,19 +130,19 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an
 			seen[ident] = depth
 			defer delete(seen, ident)
 		}
-		return redact(v.Elem(), tag, global, depth+1, seen).Addr()
+		return redact(v.Elem(), reps, tag, global, depth+1, seen).Addr()
 	case reflect.Interface:
 		if v.IsNil() {
 			return v
 		}
-		return redact(v.Elem(), tag, global, depth+1, seen)
+		return redact(v.Elem(), reps, tag, global, depth+1, seen)
 	case reflect.Array:
 		if v.Len() == 0 {
 			return v
 		}
 		r := reflect.New(v.Type()).Elem()
 		for i := 0; i < v.Len(); i++ {
-			r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen))
+			r.Index(i).Set(redact(v.Index(i), reps, tag, global, depth+1, seen))
 		}
 		return r
 	case reflect.Slice:
@@ -109,7 +165,7 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an
 		}
 		r := reflect.MakeSlice(v.Type(), v.Len(), v.Cap())
 		for i := 0; i < v.Len(); i++ {
-			r.Index(i).Set(redact(v.Index(i), tag, global, depth+1, seen))
+			r.Index(i).Set(redact(v.Index(i), reps, tag, global, depth+1, seen))
 		}
 		return r
 	case reflect.Map:
@@ -145,9 +201,13 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an
 		for it.Next() {
 			name := it.Key().String()
 			if slices.Contains(private, name) {
+				v := replaceNestedWithin(it.Value(), reps)
+				if v.IsValid() {
+					r.SetMapIndex(it.Key(), v)
+				}
 				continue
 			}
-			r.SetMapIndex(it.Key(), redact(it.Value(), tag, nextPath(name, global), depth+1, seen))
+			r.SetMapIndex(it.Key(), redact(it.Value(), reps, tag, nextPath(name, global), depth+1, seen))
 		}
 		return r
 	case reflect.Struct:
@@ -219,17 +279,82 @@ func redact(v reflect.Value, tag string, global []string, depth int, seen map[an
 				continue
 			}
 			if slices.Contains(private, names[i]) {
+				v := replaceNestedWithin(f, reps)
+				if v.IsValid() {
+					r.Field(i).Set(v)
+				}
 				continue
 			}
 			if r.Field(i).CanSet() {
-				r.Field(i).Set(redact(f, tag, nextPath(names[i], global), depth+1, seen))
+				r.Field(i).Set(redact(f, reps, tag, nextPath(names[i], global), depth+1, seen))
 			}
 		}
 		return r
 	}
 	return v
 }
 
+// replaceNestedWithin replaces deeply nested values in pointer, interface and
+// array/slice chains. If a replacement is not made an invalid reflect.Value
+// is returned. If elements are not replaced by a replacer, it is set to the
+// zero value for the type.
+func replaceNestedWithin(v reflect.Value, reps replacers) reflect.Value {
+	if len(reps) == 0 || !v.IsValid() {
+		// No replacer, or an invalid value, so fall back to removal.
+		return reflect.Value{}
+	}
+	if rep, ok := reps[v.Type()]; ok {
+		return rep(v)
+	}
+	switch v.Kind() {
+	case reflect.Pointer:
+		r := replaceNestedWithin(v.Elem(), reps)
+		if !r.IsValid() {
+			return r
+		}
+		return r.Addr()
+	case reflect.Interface:
+		r := replaceNestedWithin(v.Elem(), reps)
+		if !r.IsValid() {
+			return r
+		}
+		i := reflect.New(v.Type()).Elem()
+		i.Set(r)
+		return i
+	case reflect.Array:
+		a := reflect.New(v.Type()).Elem()
+		wasSet := false
+		for i := 0; i < v.Len(); i++ {
+			r := replaceNestedWithin(v.Index(i), reps)
+			if r.IsValid() {
+				wasSet = true
+				a.Index(i).Set(r)
+			}
+		}
+		if !wasSet {
+			return reflect.Value{}
+		}
+		return a
+	case reflect.Slice:
+		s := reflect.MakeSlice(v.Type(), v.Len(), v.Cap())
+		wasSet := false
+		for i := 0; i < v.Len(); i++ {
+			r := replaceNestedWithin(v.Index(i), reps)
+			if r.IsValid() {
+				wasSet = true
+				s.Index(i).Set(r)
+			}
+		}
+		if !wasSet {
+			return reflect.Value{}
+		}
+		return s
+	default:
+		// Could not catch, fall back to removal.
+		return reflect.Value{}
+	}
+}
+
 func nextStep(global []string) (private []string) {
 	if len(global) == 0 {
 		return nil