From 69b2d512c4ffad8a63b62739514ecc3880f19220 Mon Sep 17 00:00:00 2001 From: Raj Nishtala <113392743+rnishtala-sumo@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:45:03 -0500 Subject: [PATCH] Adding an optional replacementFormat argument to the replace_pattern editors that specified the format of the replacement string (#30837) Description: Adding an optional replacementFormat argument to the replace_pattern editors that specified the format of the replacement string Testing: Unit tests were added for this new optional argument Documentation: https://github.com/rnishtala-sumo/opentelemetry-collector-contrib/blob/ottl-replace-pattern/pkg/ottl/ottlfuncs/README.md#replace_pattern --- .chloggen/ottl-replace-pattern.yaml | 27 +++++ pkg/ottl/ottlfuncs/README.md | 18 ++- .../ottlfuncs/func_replace_all_matches.go | 18 +-- .../func_replace_all_matches_test.go | 45 ++++--- .../ottlfuncs/func_replace_all_patterns.go | 19 +-- .../func_replace_all_patterns_test.go | 113 +++++++++++++---- pkg/ottl/ottlfuncs/func_replace_match.go | 18 +-- pkg/ottl/ottlfuncs/func_replace_match_test.go | 43 ++++--- pkg/ottl/ottlfuncs/func_replace_pattern.go | 46 +++++-- .../ottlfuncs/func_replace_pattern_test.go | 114 +++++++++++++++--- 10 files changed, 350 insertions(+), 111 deletions(-) create mode 100755 .chloggen/ottl-replace-pattern.yaml diff --git a/.chloggen/ottl-replace-pattern.yaml b/.chloggen/ottl-replace-pattern.yaml new file mode 100755 index 000000000000..055d5e6a4d35 --- /dev/null +++ b/.chloggen/ottl-replace-pattern.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support to specify the format for a replacement string + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [27820] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index fc6f2df0476d..e97efc699335 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -251,17 +251,19 @@ Examples: The `replace_all_matches` function replaces any matching string value with the replacement string. `target` is a path expression to a `pcommon.Map` type field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching string with the hash value of `replacement`. +`replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. Each string value in `target` that matches `pattern` will get replaced with `replacement`. Non-string values are ignored. Examples: - `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}")` -- `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` +- `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256, "/user/%s")` ### replace_all_patterns `replace_all_patterns(target, mode, regex, replacement, Optional[function])` +`replace_all_patterns(target, mode, regex, replacement, function, replacementFormat)` The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -271,7 +273,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -280,7 +282,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.")` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", SHA256, "k8s.%s")` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -293,6 +296,7 @@ If using OTTL outside of collector configuration, `$` should not be escaped and The `replace_match` function allows replacing entire strings if they match a glob pattern. `target` is a path expression to a telemetry field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. +`replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. If `target` matches `pattern` it will get replaced with `replacement`. @@ -301,11 +305,13 @@ The `function` is an optional argument that can take in any Converter that accep Examples: - `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}")` -- `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` +- `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256, "/user/%s")` ### replace_pattern `replace_pattern(target, regex, replacement, Optional[function])` +`replace_pattern(target, regex, replacement, function)` +`replace_pattern(target, regex, replacement, function, replacementFormat)` The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -313,7 +319,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. @@ -321,7 +327,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", SHA256, "k8s.%s")` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches.go b/pkg/ottl/ottlfuncs/func_replace_all_matches.go index 597cf878a12c..1e4fc2874aa8 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches.go @@ -14,10 +14,11 @@ import ( ) type ReplaceAllMatchesArguments[K any] struct { - Target ottl.PMapGetter[K] - Pattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.PMapGetter[K] + Pattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replaceAllMatchesFuncArgs[K any] struct { @@ -35,10 +36,10 @@ func createReplaceAllMatchesFunction[K any](_ ottl.FunctionContext, oArgs ottl.A return nil, fmt.Errorf("ReplaceAllMatchesFactory args must be of type *ReplaceAllMatchesArguments[K]") } - return replaceAllMatches(args.Target, args.Pattern, args.Replacement, args.Function) + return replaceAllMatches(args.Target, args.Pattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) @@ -68,7 +69,10 @@ func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replace if !ok { return nil, fmt.Errorf("replacement value is not a string") } - replacementVal = replacementValStr + replacementVal, err = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if err != nil { + return nil, err + } } val.Range(func(key string, value pcommon.Value) bool { if glob.Match(value.Str()) { diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go index 3494aa0d4866..0ca6fc1dcdbb 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go @@ -27,6 +27,11 @@ func Test_replaceAllMatches(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Map](), } + prefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=%s", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) target := &ottl.StandardPMapGetter[pcommon.Map]{ @@ -36,12 +41,13 @@ func Test_replaceAllMatches(t *testing.T) { } tests := []struct { - name string - target ottl.PMapGetter[pcommon.Map] - pattern string - replacement ottl.StringGetter[pcommon.Map] - function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] - want func(pcommon.Map) + name string + target ottl.PMapGetter[pcommon.Map] + pattern string + replacement ottl.StringGetter[pcommon.Map] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Map]] + want func(pcommon.Map) }{ { name: "replace only matches (with hash function)", @@ -52,10 +58,11 @@ func Test_replaceAllMatches(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + function: optionalArg, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](prefix), want: func(expectedMap pcommon.Map) { - expectedMap.PutStr("test", "hash(hello {universe})") - expectedMap.PutStr("test2", "hash(hello {universe})") + expectedMap.PutStr("test", "prefix=hash(hello {universe})") + expectedMap.PutStr("test2", "prefix=hash(hello {universe})") expectedMap.PutStr("test3", "goodbye") }, }, @@ -68,7 +75,8 @@ func Test_replaceAllMatches(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe}") expectedMap.PutStr("test2", "hello {universe}") @@ -84,7 +92,8 @@ func Test_replaceAllMatches(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -97,7 +106,7 @@ func Test_replaceAllMatches(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioMap) @@ -125,8 +134,9 @@ func Test_replaceAllMatches_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "*", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, input) assert.Error(t, err) @@ -145,8 +155,9 @@ func Test_replaceAllMatches_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -174,8 +185,9 @@ func Test_replaceAllMatches_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -195,8 +207,9 @@ func Test_replaceAllMatches_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "*", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, nil) assert.Error(t, err) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 461741b90b11..0f3aae69e8c2 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -19,11 +19,12 @@ const ( ) type ReplaceAllPatternsArguments[K any] struct { - Target ottl.PMapGetter[K] - Mode string - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.PMapGetter[K] + Mode string + RegexPattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] { @@ -37,10 +38,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) @@ -66,7 +67,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt case modeValue: if compiledPattern.MatchString(originalValue.Str()) { if !fn.IsEmpty() { - updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal) + updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal, replacementFormat) if err != nil { return false } @@ -81,7 +82,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt case modeKey: if compiledPattern.MatchString(key) { if !fn.IsEmpty() { - updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal) + updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal, replacementFormat) if err != nil { return false } diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index a83335d44605..896926a4046f 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -30,6 +30,16 @@ func Test_replaceAllPatterns(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Map](), } + prefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=%s", nil + }, + } + invalidPrefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) target := &ottl.StandardPMapGetter[pcommon.Map]{ @@ -39,13 +49,14 @@ func Test_replaceAllPatterns(t *testing.T) { } tests := []struct { - name string - target ottl.PMapGetter[pcommon.Map] - mode string - pattern string - replacement ottl.StringGetter[pcommon.Map] - function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] - want func(pcommon.Map) + name string + target ottl.PMapGetter[pcommon.Map] + mode string + pattern string + replacement ottl.StringGetter[pcommon.Map] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Map]] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] + want func(pcommon.Map) }{ { name: "replace only matches (with hash function)", @@ -57,7 +68,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hash(hello {universe}) world") expectedMap.PutStr("test2", "hash(hello {universe})") @@ -127,6 +139,44 @@ func Test_replaceAllPatterns(t *testing.T) { expectedMap.PutBool("test6", true) }, }, + { + name: "replace only matches (with replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](prefix), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "prefix=hash(hello {universe}) world") + expectedMap.PutStr("test2", "prefix=hash(hello {universe})") + expectedMap.PutStr("test3", "goodbye world1 and world2") + expectedMap.PutInt("test4", 1234) + expectedMap.PutDouble("test5", 1234) + expectedMap.PutBool("test6", true) + }, + }, + { + name: "replace only matches (with invalid replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](invalidPrefix), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutEmpty("test") + expectedMap.Remove("test") + }, + }, { name: "replace only matches", target: target, @@ -137,7 +187,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") expectedMap.PutStr("test2", "hello {universe}") @@ -157,7 +208,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -177,7 +229,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") expectedMap.PutStr("test2", "hello") @@ -297,7 +350,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -318,7 +372,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -339,7 +394,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test.", "hello world") @@ -360,7 +416,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -381,7 +438,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test-2", "hello") @@ -401,7 +459,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -418,7 +477,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -445,8 +504,9 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, input) @@ -466,8 +526,9 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -495,8 +556,9 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -516,8 +578,9 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -537,9 +600,10 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, replacementFormat) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") assert.Nil(t, exprFunc) @@ -558,9 +622,10 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, replacementFormat) assert.Nil(t, exprFunc) assert.Contains(t, err.Error(), "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_match.go b/pkg/ottl/ottlfuncs/func_replace_match.go index 123320caa421..21cf825234bc 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match.go +++ b/pkg/ottl/ottlfuncs/func_replace_match.go @@ -13,10 +13,11 @@ import ( ) type ReplaceMatchArguments[K any] struct { - Target ottl.GetSetter[K] - Pattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.GetSetter[K] + Pattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replaceMatchFuncArgs[K any] struct { @@ -34,10 +35,10 @@ func createReplaceMatchFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argume return nil, fmt.Errorf("ReplaceMatchFactory args must be of type *ReplaceMatchArguments[K]") } - return replaceMatch(args.Target, args.Pattern, args.Replacement, args.Function) + return replaceMatch(args.Target, args.Pattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) @@ -67,7 +68,10 @@ func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement o if !ok { return nil, fmt.Errorf("replacement value is not a string") } - replacementVal = replacementValStr + replacementVal, err = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if err != nil { + return nil, err + } } if err != nil { return nil, err diff --git a/pkg/ottl/ottlfuncs/func_replace_match_test.go b/pkg/ottl/ottlfuncs/func_replace_match_test.go index a61e28560f90..d390d4e3165c 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_match_test.go @@ -23,6 +23,11 @@ func Test_replaceMatch(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Value](), } + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=%s", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { @@ -35,12 +40,13 @@ func Test_replaceMatch(t *testing.T) { } tests := []struct { - name string - target ottl.GetSetter[pcommon.Value] - pattern string - replacement ottl.StringGetter[pcommon.Value] - function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] - want func(pcommon.Value) + name string + target ottl.GetSetter[pcommon.Value] + pattern string + replacement ottl.StringGetter[pcommon.Value] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Value]] + want func(pcommon.Value) }{ { name: "replace match (with hash function)", @@ -51,9 +57,10 @@ func Test_replaceMatch(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + function: optionalArg, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix), want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("hash(hello {universe})") + expectedValue.SetStr("passwd=hash(hello {universe})") }, }, { @@ -65,7 +72,8 @@ func Test_replaceMatch(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello {universe}") }, @@ -79,7 +87,8 @@ func Test_replaceMatch(t *testing.T) { return "goodbye {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello world") }, @@ -89,7 +98,7 @@ func Test_replaceMatch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) assert.NoError(t, err) @@ -120,8 +129,9 @@ func Test_replaceMatch_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "*", replacement, function) + exprFunc, err := replaceMatch[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -148,8 +158,9 @@ func Test_replaceMatch_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "regexp", replacement, function) + exprFunc, err := replaceMatch[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -181,8 +192,9 @@ func Test_replaceMatch_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "regexp", replacement, function) + exprFunc, err := replaceMatch[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -207,8 +219,9 @@ func Test_replaceMatch_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "*", replacement, function) + exprFunc, err := replaceMatch[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, nil) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 7b02b3896ea1..b77d3463d19b 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -13,10 +13,11 @@ import ( ) type ReplacePatternArguments[K any] struct { - Target ottl.GetSetter[K] - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.GetSetter[K] + RegexPattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replacePatternFuncArgs[K any] struct { @@ -34,10 +35,35 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string) (string, error) { +func validFormatString(formatString string) bool { + // Check for exactly one %s and no other invalid format specifiers + validPattern := `^(.*?%s.*?)$` + validRegex := regexp.MustCompile(validPattern) + invalidPattern := `%[^s]` + invalidRegex := regexp.MustCompile(invalidPattern) + + return validRegex.MatchString(formatString) && !invalidRegex.MatchString(formatString) +} + +func applyReplaceFormat[K any](ctx context.Context, tCtx K, replacementFormat ottl.Optional[ottl.StringGetter[K]], replacementVal string) (string, error) { + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + formatStringVal, errFmt := formatString.Get(ctx, tCtx) + if errFmt != nil { + return "", errFmt + } + if !validFormatString(formatStringVal) { + return "", fmt.Errorf("replacementFormat must be format string containing a single %%s and no other format specifiers") + } + replacementVal = fmt.Sprintf(formatStringVal, replacementVal) + } + return replacementVal, nil +} + +func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string, replacementFormat ottl.Optional[ottl.StringGetter[K]]) (string, error) { var updatedString string updatedString = originalValStr submatches := compiledPattern.FindAllStringSubmatchIndex(updatedString, -1) @@ -62,12 +88,16 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern if !ok { return "", fmt.Errorf("the replacement value must be a string") } + replacementValStr, errNew = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if errNew != nil { + return "", errNew + } updatedString = strings.ReplaceAll(updatedString, fullMatch, replacementValStr) } return updatedString, nil } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) @@ -89,7 +119,7 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac if compiledPattern.MatchString(originalValStr) { if !fn.IsEmpty() { var updatedString string - updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal) + updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal, replacementFormat) if err != nil { return nil, err } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index e14306b57890..895eda7fe4e5 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -53,6 +53,16 @@ func Test_replacePattern(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Value](), } + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=%s", nil + }, + } + passwdSuffix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "%s (passwd)", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { @@ -65,12 +75,13 @@ func Test_replacePattern(t *testing.T) { } tests := []struct { - name string - target ottl.GetSetter[pcommon.Value] - pattern string - replacement ottl.StringGetter[pcommon.Value] - function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] - want func(pcommon.Value) + name string + target ottl.GetSetter[pcommon.Value] + pattern string + replacement ottl.StringGetter[pcommon.Value] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Value]] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] + want func(pcommon.Value) }{ { name: "replace regex match (with hash function)", @@ -81,9 +92,10 @@ func Test_replacePattern(t *testing.T) { return "$1", nil }, }, - function: optionalArg, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix), + function: optionalArg, want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("application hash(sensitivedtata)otherarg=notsensitive key1 key2") + expectedValue.SetStr("application passwd=hash(sensitivedtata)otherarg=notsensitive key1 key2") }, }, { @@ -128,6 +140,21 @@ func Test_replacePattern(t *testing.T) { expectedValue.SetStr("application otherarg=notsensitive key1 key2") }, }, + { + name: "replace regex match (with hash function)", + target: target, + pattern: `passwd\=[^\s]*`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=***", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdSuffix), + function: optionalArg, + want: func(expectedValue pcommon.Value) { + expectedValue.SetStr("application hash(passwd=***) (passwd) otherarg=notsensitive key1 key2") + }, + }, { name: "replace regex match", target: target, @@ -137,7 +164,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") }, @@ -151,7 +179,8 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") }, @@ -165,7 +194,8 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") }, @@ -179,7 +209,8 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") }, @@ -193,7 +224,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") }, @@ -202,7 +234,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -213,6 +245,7 @@ func Test_replacePattern(t *testing.T) { tt.want(expected) assert.Equal(t, expected, scenarioValue) + }) } } @@ -234,8 +267,9 @@ func Test_replacePattern_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -261,8 +295,9 @@ func Test_replacePattern_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -294,8 +329,9 @@ func Test_replacePattern_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -320,8 +356,9 @@ func Test_replacePattern_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function) + exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -346,9 +383,48 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidRegexPattern := "*" - _, err := replacePattern[any](target, invalidRegexPattern, replacement, function) + _, err := replacePattern[any](target, invalidRegexPattern, replacement, function, replacementFormat) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") } + +func Test_replacePattern_bad_format_string(t *testing.T) { + input := pcommon.NewValueStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") + target := &ottl.StandardGetSetter[pcommon.Value]{ + Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { + return tCtx.Str(), nil + }, + Setter: func(ctx context.Context, tCtx pcommon.Value, val any) error { + tCtx.SetStr(val.(string)) + return nil + }, + } + replacement := &ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=*** ", nil + }, + } + ottlValue := ottl.StandardFunctionGetter[pcommon.Value]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Value]()["SHA256"], + } + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=", nil + }, + } + replacementFormat := ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix) // This is not a valid format string + function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) + + exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, function, replacementFormat) + assert.NoError(t, err) + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "replacementFormat must be format string containing a single %s") + assert.Nil(t, result) +}