From ba84f677ef33742b55d84fbd1a104526caa2acec Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Mon, 16 Oct 2023 12:54:24 -0400 Subject: [PATCH 1/6] feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors --- pkg/ottl/ottlfuncs/README.md | 17 +++++-- .../ottlfuncs/func_replace_all_patterns.go | 15 +++--- .../func_replace_all_patterns_test.go | 47 ++++++++++++------- pkg/ottl/ottlfuncs/func_replace_pattern.go | 11 +++-- .../ottlfuncs/func_replace_pattern_test.go | 33 ++++++++----- 5 files changed, 76 insertions(+), 47 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index fc6f2df0476d..b6ef50f13a1c 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -261,7 +261,11 @@ Examples: ### replace_all_patterns +<<<<<<< HEAD `replace_all_patterns(target, mode, regex, replacement, Optional[function])` +======= +`replace_all_patterns(target, mode, regex, replacement, replacementPrefix, function)` +>>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -271,7 +275,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -280,7 +284,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -305,7 +310,11 @@ Examples: ### replace_pattern +<<<<<<< HEAD `replace_pattern(target, regex, replacement, Optional[function])` +======= +`replace_pattern(target, regex, replacement, replacementPrefix, function)` +>>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -313,7 +322,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. @@ -321,7 +330,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 461741b90b11..783d91710200 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -19,11 +19,12 @@ const ( ) type ReplaceAllPatternsArguments[K any] struct { - Target ottl.PMapGetter[K] - Mode string - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.PMapGetter[K] + Mode string + RegexPattern string + Replacement ottl.StringGetter[K] + ReplacementPrefix ottl.Optional[string] + Function ottl.Optional[ottl.FunctionGetter[K]] } func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] { @@ -37,10 +38,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index a83335d44605..2586b650609f 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -39,13 +39,14 @@ func Test_replaceAllPatterns(t *testing.T) { } tests := []struct { - name string - target ottl.PMapGetter[pcommon.Map] - mode string - pattern string - replacement ottl.StringGetter[pcommon.Map] - function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] - want func(pcommon.Map) + name string + target ottl.PMapGetter[pcommon.Map] + mode string + pattern string + replacement ottl.StringGetter[pcommon.Map] + replacementPrefix ottl.Optional[string] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] + want func(pcommon.Map) }{ { name: "replace only matches (with hash function)", @@ -57,7 +58,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + replacementPrefix: ottl.Optional[string]{}, + function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hash(hello {universe}) world") expectedMap.PutStr("test2", "hash(hello {universe})") @@ -137,7 +139,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") expectedMap.PutStr("test2", "hello {universe}") @@ -157,7 +160,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -177,7 +181,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") expectedMap.PutStr("test2", "hello") @@ -297,7 +302,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -318,7 +324,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -339,7 +346,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test.", "hello world") @@ -360,7 +368,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -381,7 +390,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test-2", "hello") @@ -401,7 +411,8 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() expectedMap.PutStr("test", "hello world") @@ -418,7 +429,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 7b02b3896ea1..8d7bdeb76765 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -13,10 +13,11 @@ import ( ) type ReplacePatternArguments[K any] struct { - Target ottl.GetSetter[K] - RegexPattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.GetSetter[K] + RegexPattern string + Replacement ottl.StringGetter[K] + ReplacementPrefix ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value + Function ottl.Optional[ottl.FunctionGetter[K]] } type replacePatternFuncArgs[K any] struct { @@ -34,7 +35,7 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) } func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string) (string, error) { diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index e14306b57890..49fa2ba61163 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -65,12 +65,13 @@ func Test_replacePattern(t *testing.T) { } tests := []struct { - name string - target ottl.GetSetter[pcommon.Value] - pattern string - replacement ottl.StringGetter[pcommon.Value] - function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] - want func(pcommon.Value) + name string + target ottl.GetSetter[pcommon.Value] + pattern string + replacement ottl.StringGetter[pcommon.Value] + replacementPrefix ottl.Optional[string] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] + want func(pcommon.Value) }{ { name: "replace regex match (with hash function)", @@ -81,7 +82,8 @@ func Test_replacePattern(t *testing.T) { return "$1", nil }, }, - function: optionalArg, + replacementPrefix: ottl.NewTestingOptional[string]("passwd="), + function: optionalArg, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application hash(sensitivedtata)otherarg=notsensitive key1 key2") }, @@ -137,7 +139,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") }, @@ -151,7 +154,8 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") }, @@ -165,7 +169,8 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") }, @@ -179,7 +184,8 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") }, @@ -193,7 +199,8 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementPrefix: ottl.Optional[string]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") }, @@ -202,7 +209,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) From 47730162217557f1fe3cea3a38d13d969a1c9b8f Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Wed, 18 Oct 2023 17:39:44 -0400 Subject: [PATCH 2/6] Change the replacement prefix string to be a format string, this allows for suffixes --- pkg/ottl/ottlfuncs/README.md | 18 +++-- .../ottlfuncs/func_replace_all_patterns.go | 6 +- .../func_replace_all_patterns_test.go | 62 ++++++++++++++---- pkg/ottl/ottlfuncs/func_replace_pattern.go | 4 +- .../ottlfuncs/func_replace_pattern_test.go | 65 ++++++++++++++++--- 5 files changed, 125 insertions(+), 30 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index b6ef50f13a1c..8e1d018c1330 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -261,11 +261,15 @@ Examples: ### replace_all_patterns +<<<<<<< HEAD <<<<<<< HEAD `replace_all_patterns(target, mode, regex, replacement, Optional[function])` ======= `replace_all_patterns(target, mode, regex, replacement, replacementPrefix, function)` >>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) +======= +`replace_all_patterns(target, mode, regex, replacement, replacementFormat, function)` +>>>>>>> ee00bae6fe (Change the replacement prefix string to be a format string, this allows for suffixes) The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -275,7 +279,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -284,8 +288,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.)` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s")` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -310,11 +314,15 @@ Examples: ### replace_pattern +<<<<<<< HEAD <<<<<<< HEAD `replace_pattern(target, regex, replacement, Optional[function])` ======= `replace_pattern(target, regex, replacement, replacementPrefix, function)` >>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) +======= +`replace_pattern(target, regex, replacement, replacementFormat, function)` +>>>>>>> ee00bae6fe (Change the replacement prefix string to be a format string, this allows for suffixes) The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -322,7 +330,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementPrefix` is an optional string argument to prefix a `replacement` string with. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. @@ -330,7 +338,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 783d91710200..90b2e5271d02 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -23,7 +23,7 @@ type ReplaceAllPatternsArguments[K any] struct { Mode string RegexPattern string Replacement ottl.StringGetter[K] - ReplacementPrefix ottl.Optional[string] + ReplacementFormat ottl.Optional[string] Function ottl.Optional[ottl.FunctionGetter[K]] } @@ -38,10 +38,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementPrefix ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementFormat ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index 2586b650609f..a24e05b497de 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -44,7 +44,7 @@ func Test_replaceAllPatterns(t *testing.T) { mode string pattern string replacement ottl.StringGetter[pcommon.Map] - replacementPrefix ottl.Optional[string] + replacementFormat ottl.Optional[string] function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] want func(pcommon.Map) }{ @@ -58,7 +58,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hash(hello {universe}) world") @@ -129,6 +129,44 @@ func Test_replaceAllPatterns(t *testing.T) { expectedMap.PutBool("test6", true) }, }, + { + name: "replace only matches (with replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") + expectedMap.PutStr("test2", "passwd=passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") // replacement format is applied twice + expectedMap.PutStr("test3", "goodbye world1 and world2") + expectedMap.PutInt("test4", 1234) + expectedMap.PutDouble("test5", 1234) + expectedMap.PutBool("test6", true) + }, + }, + { + name: "replace only matches (with invalid replacement format)", + target: target, + mode: modeValue, + pattern: "hello", + replacement: ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (interface{}, error) { + return "hello {universe}", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("passwd="), + function: optionalArg, + want: func(expectedMap pcommon.Map) { + expectedMap.PutEmpty("test") + expectedMap.Remove("test") + }, + }, { name: "replace only matches", target: target, @@ -139,7 +177,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") @@ -160,7 +198,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -181,7 +219,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") @@ -302,7 +340,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -324,7 +362,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -346,7 +384,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -368,7 +406,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -390,7 +428,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -411,7 +449,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -429,7 +467,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 8d7bdeb76765..39a3efd7c699 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -16,7 +16,7 @@ type ReplacePatternArguments[K any] struct { Target ottl.GetSetter[K] RegexPattern string Replacement ottl.StringGetter[K] - ReplacementPrefix ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value + ReplacementFormat ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value Function ottl.Optional[ottl.FunctionGetter[K]] } @@ -35,7 +35,7 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementPrefix, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) } func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string) (string, error) { diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 49fa2ba61163..e837ab760c8f 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -69,7 +69,7 @@ func Test_replacePattern(t *testing.T) { target ottl.GetSetter[pcommon.Value] pattern string replacement ottl.StringGetter[pcommon.Value] - replacementPrefix ottl.Optional[string] + replacementFormat ottl.Optional[string] function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] want func(pcommon.Value) }{ @@ -82,7 +82,7 @@ func Test_replacePattern(t *testing.T) { return "$1", nil }, }, - replacementPrefix: ottl.NewTestingOptional[string]("passwd="), + replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), function: optionalArg, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application hash(sensitivedtata)otherarg=notsensitive key1 key2") @@ -130,6 +130,21 @@ func Test_replacePattern(t *testing.T) { expectedValue.SetStr("application otherarg=notsensitive key1 key2") }, }, + { + name: "replace regex match (with hash function)", + target: target, + pattern: `passwd\=[^\s]*`, + replacement: ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + }, + replacementFormat: ottl.NewTestingOptional[string]("%s (passwd)"), + function: optionalArg, + want: func(expectedValue pcommon.Value) { + expectedValue.SetStr("application 0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838be (passwd) otherarg=notsensitive key1 key2") + }, + }, { name: "replace regex match", target: target, @@ -139,7 +154,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") @@ -154,7 +169,7 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") @@ -169,7 +184,7 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") @@ -184,7 +199,7 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") @@ -199,7 +214,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - replacementPrefix: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[string]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") @@ -209,7 +224,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementPrefix, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -220,6 +235,7 @@ func Test_replacePattern(t *testing.T) { tt.want(expected) assert.Equal(t, expected, scenarioValue) + }) } } @@ -359,3 +375,36 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") } + +func Test_replacePattern_bad_format_string(t *testing.T) { + input := pcommon.NewValueStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") + target := &ottl.StandardGetSetter[pcommon.Value]{ + Getter: func(ctx context.Context, tCtx pcommon.Value) (interface{}, error) { + return tCtx.Str(), nil + }, + Setter: func(ctx context.Context, tCtx pcommon.Value, val interface{}) error { + tCtx.SetStr(val.(string)) + return nil + }, + } + replacement := &ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (interface{}, error) { + return "passwd=*** ", nil + }, + } + ottlValue := ottl.StandardFunctionGetter[pcommon.Value]{ + FCtx: ottl.FunctionContext{ + Set: componenttest.NewNopTelemetrySettings(), + }, + Fact: StandardConverters[pcommon.Value]()["SHA256"], + } + replacementFormat := ottl.NewTestingOptional[string]("passwd=") // This is not a valid format string + function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) + + exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, replacementFormat, function) + assert.NoError(t, err) + result, err := exprFunc(nil, input) + require.Error(t, err) + assert.ErrorContains(t, err, "replacementFormat must be format string with %s") + assert.Nil(t, result) +} From 3d24c343ec1e00055c9214da2d43ccd17db1eff1 Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Mon, 29 Jan 2024 15:08:02 -0500 Subject: [PATCH 3/6] Updating to PR to include the latest changes from main --- .chloggen/ottl-replace-pattern.yaml | 27 +++++++++++++++ pkg/ottl/ottlfuncs/README.md | 25 ++++---------- .../ottlfuncs/func_replace_all_patterns.go | 10 +++--- .../func_replace_all_patterns_test.go | 30 ++++++++++------- pkg/ottl/ottlfuncs/func_replace_pattern.go | 17 +++++++--- .../ottlfuncs/func_replace_pattern_test.go | 33 +++++++++++-------- 6 files changed, 87 insertions(+), 55 deletions(-) create mode 100755 .chloggen/ottl-replace-pattern.yaml diff --git a/.chloggen/ottl-replace-pattern.yaml b/.chloggen/ottl-replace-pattern.yaml new file mode 100755 index 000000000000..055d5e6a4d35 --- /dev/null +++ b/.chloggen/ottl-replace-pattern.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support to specify the format for a replacement string + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [27820] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 8e1d018c1330..157a197e8e23 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -261,15 +261,8 @@ Examples: ### replace_all_patterns -<<<<<<< HEAD -<<<<<<< HEAD `replace_all_patterns(target, mode, regex, replacement, Optional[function])` -======= -`replace_all_patterns(target, mode, regex, replacement, replacementPrefix, function)` ->>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) -======= -`replace_all_patterns(target, mode, regex, replacement, replacementFormat, function)` ->>>>>>> ee00bae6fe (Change the replacement prefix string to be a format string, this allows for suffixes) +`replace_all_patterns(target, mode, regex, replacement, function, replacementFormat)` The `replace_all_patterns` function replaces any segments in a string value or key that match the regex pattern with the replacement string. @@ -288,8 +281,8 @@ Examples: - `replace_all_patterns(attributes, "value", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "/account/\\d{4}", "/account/{accountId}")` - `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s")` -- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.")` +- `replace_all_patterns(attributes, "key", "^kube_([0-9A-Za-z]+_)", "$$1.", SHA256, "k8s.%s")` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. @@ -314,15 +307,9 @@ Examples: ### replace_pattern -<<<<<<< HEAD -<<<<<<< HEAD `replace_pattern(target, regex, replacement, Optional[function])` -======= -`replace_pattern(target, regex, replacement, replacementPrefix, function)` ->>>>>>> 336226dd65 (feat(pkg/ottl) Adding an optional replacement prefix argument to the replace_pattern editors) -======= -`replace_pattern(target, regex, replacement, replacementFormat, function)` ->>>>>>> ee00bae6fe (Change the replacement prefix string to be a format string, this allows for suffixes) +`replace_pattern(target, regex, replacement, function)` +`replace_pattern(target, regex, replacement, function, replacementFormat)` The `replace_pattern` function allows replacing all string sections that match a regex pattern with a new value. @@ -338,7 +325,7 @@ Examples: - `replace_pattern(resource.attributes["process.command_line"], "password\\=[^\\s]*(\\s?)", "password=***")` - `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "k8s.$$1.")` -- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", "k8s.%s", SHA256)` +- `replace_pattern(name, "^kube_([0-9A-Za-z]+_)", "$$1.", SHA256, "k8s.%s")` Note that when using OTTL within the collector's configuration file, `$` must be escaped to `$$` to bypass environment variable substitution logic. To input a literal `$` from the configuration file, use `$$$`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index 90b2e5271d02..c4853dcb9ed7 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -23,8 +23,8 @@ type ReplaceAllPatternsArguments[K any] struct { Mode string RegexPattern string Replacement ottl.StringGetter[K] - ReplacementFormat ottl.Optional[string] Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[string] } func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] { @@ -38,10 +38,10 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return nil, fmt.Errorf("ReplaceAllPatternsFactory args must be of type *ReplaceAllPatternsArguments[K]") } - return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) + return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], replacementFormat ottl.Optional[string], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[string]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) @@ -67,7 +67,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt case modeValue: if compiledPattern.MatchString(originalValue.Str()) { if !fn.IsEmpty() { - updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal) + updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, originalValue.Str(), replacementVal, replacementFormat) if err != nil { return false } @@ -82,7 +82,7 @@ func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPatt case modeKey: if compiledPattern.MatchString(key) { if !fn.IsEmpty() { - updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal) + updatedString, err := applyOptReplaceFunction(ctx, tCtx, compiledPattern, fn, key, replacementVal, replacementFormat) if err != nil { return false } diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index a24e05b497de..a6b3101a1691 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -135,15 +135,15 @@ func Test_replaceAllPatterns(t *testing.T) { mode: modeValue, pattern: "hello", replacement: ottl.StandardStringGetter[pcommon.Map]{ - Getter: func(context.Context, pcommon.Map) (interface{}, error) { + Getter: func(context.Context, pcommon.Map) (any, error) { return "hello {universe}", nil }, }, - replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), + replacementFormat: ottl.NewTestingOptional[string]("prefix=%s"), function: optionalArg, want: func(expectedMap pcommon.Map) { - expectedMap.PutStr("test", "passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad world") - expectedMap.PutStr("test2", "passwd=passwd=4804d6b7f03268e33f78c484977f3d81771220df07cc6aac4ad4868102141fad") // replacement format is applied twice + expectedMap.PutStr("test", "prefix=hash(hello {universe}) world") + expectedMap.PutStr("test2", "prefix=hash(hello {universe})") // replacement format is applied twice expectedMap.PutStr("test3", "goodbye world1 and world2") expectedMap.PutInt("test4", 1234) expectedMap.PutDouble("test5", 1234) @@ -156,7 +156,7 @@ func Test_replaceAllPatterns(t *testing.T) { mode: modeValue, pattern: "hello", replacement: ottl.StandardStringGetter[pcommon.Map]{ - Getter: func(context.Context, pcommon.Map) (interface{}, error) { + Getter: func(context.Context, pcommon.Map) (any, error) { return "hello {universe}", nil }, }, @@ -467,7 +467,7 @@ func Test_replaceAllPatterns(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) + exprFunc, err := replaceAllPatterns[pcommon.Map](tt.target, tt.mode, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, scenarioMap) @@ -494,8 +494,9 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, input) @@ -515,8 +516,9 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -544,8 +546,9 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -565,8 +568,9 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, nil) @@ -586,9 +590,10 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} invalidRegexPattern := "*" - exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function) + exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, replacementFormat) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") assert.Nil(t, exprFunc) @@ -607,9 +612,10 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} invalidMode := "invalid" - exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function) + exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, replacementFormat) assert.Nil(t, exprFunc) assert.Contains(t, err.Error(), "invalid mode") } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 39a3efd7c699..2d43dc78ffc2 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -16,8 +16,8 @@ type ReplacePatternArguments[K any] struct { Target ottl.GetSetter[K] RegexPattern string Replacement ottl.StringGetter[K] - ReplacementFormat ottl.Optional[string] // ReplacementPrefix is an optional prefix to add to the replacement value Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[string] } type replacePatternFuncArgs[K any] struct { @@ -35,10 +35,10 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return nil, fmt.Errorf("ReplacePatternFactory args must be of type *ReplacePatternArguments[K]") } - return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.ReplacementFormat, args.Function) + return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string) (string, error) { +func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string, replacementFormat ottl.Optional[string]) (string, error) { var updatedString string updatedString = originalValStr submatches := compiledPattern.FindAllStringSubmatchIndex(updatedString, -1) @@ -63,12 +63,19 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern if !ok { return "", fmt.Errorf("the replacement value must be a string") } + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + if !strings.Contains(formatString, "%s") { + return "", fmt.Errorf("replacementFormat must be format string with %%s") + } + replacementValStr = fmt.Sprintf(replacementFormat.Get(), replacementValStr) + } updatedString = strings.ReplaceAll(updatedString, fullMatch, replacementValStr) } return updatedString, nil } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[string]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) @@ -90,7 +97,7 @@ func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replac if compiledPattern.MatchString(originalValStr) { if !fn.IsEmpty() { var updatedString string - updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal) + updatedString, err = applyOptReplaceFunction[K](ctx, tCtx, compiledPattern, fn, originalValStr, replacementVal, replacementFormat) if err != nil { return nil, err } diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index e837ab760c8f..4393e1ef28ec 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -85,7 +85,7 @@ func Test_replacePattern(t *testing.T) { replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), function: optionalArg, want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("application hash(sensitivedtata)otherarg=notsensitive key1 key2") + expectedValue.SetStr("application passwd=hash(sensitivedtata)otherarg=notsensitive key1 key2") }, }, { @@ -135,14 +135,14 @@ func Test_replacePattern(t *testing.T) { target: target, pattern: `passwd\=[^\s]*`, replacement: ottl.StandardStringGetter[pcommon.Value]{ - Getter: func(context.Context, pcommon.Value) (interface{}, error) { - return "passwd=*** ", nil + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=***", nil }, }, replacementFormat: ottl.NewTestingOptional[string]("%s (passwd)"), function: optionalArg, want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("application 0f2407f2d83337b1f757eb1754a7643ce0e8fba620bc605c54566cd6dfd838be (passwd) otherarg=notsensitive key1 key2") + expectedValue.SetStr("application hash(passwd=***) (passwd) otherarg=notsensitive key1 key2") }, }, { @@ -224,7 +224,7 @@ func Test_replacePattern(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.replacementFormat, tt.function) + exprFunc, err := replacePattern(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) @@ -257,8 +257,9 @@ func Test_replacePattern_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -284,8 +285,9 @@ func Test_replacePattern_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -317,8 +319,9 @@ func Test_replacePattern_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replacePattern[any](target, "regexp", replacement, function) + exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -343,8 +346,9 @@ func Test_replacePattern_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} - exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function) + exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, nil) @@ -369,9 +373,10 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[string]{} invalidRegexPattern := "*" - _, err := replacePattern[any](target, invalidRegexPattern, replacement, function) + _, err := replacePattern[any](target, invalidRegexPattern, replacement, function, replacementFormat) require.Error(t, err) assert.ErrorContains(t, err, "error parsing regexp:") } @@ -379,16 +384,16 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { func Test_replacePattern_bad_format_string(t *testing.T) { input := pcommon.NewValueStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") target := &ottl.StandardGetSetter[pcommon.Value]{ - Getter: func(ctx context.Context, tCtx pcommon.Value) (interface{}, error) { + Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { return tCtx.Str(), nil }, - Setter: func(ctx context.Context, tCtx pcommon.Value, val interface{}) error { + Setter: func(ctx context.Context, tCtx pcommon.Value, val any) error { tCtx.SetStr(val.(string)) return nil }, } replacement := &ottl.StandardStringGetter[pcommon.Value]{ - Getter: func(context.Context, pcommon.Value) (interface{}, error) { + Getter: func(context.Context, pcommon.Value) (any, error) { return "passwd=*** ", nil }, } @@ -401,7 +406,7 @@ func Test_replacePattern_bad_format_string(t *testing.T) { replacementFormat := ottl.NewTestingOptional[string]("passwd=") // This is not a valid format string function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) - exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, replacementFormat, function) + exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) require.Error(t, err) From 7bb59d3022c4030a956b080c9cc9230c657b58f5 Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Thu, 1 Feb 2024 16:39:02 -0500 Subject: [PATCH 4/6] Change the replaceFormat string to type StringGetter --- .../ottlfuncs/func_replace_all_patterns.go | 4 +- .../func_replace_all_patterns_test.go | 50 +++++++++++-------- pkg/ottl/ottlfuncs/func_replace_pattern.go | 16 +++--- .../ottlfuncs/func_replace_pattern_test.go | 45 +++++++++++------ 4 files changed, 72 insertions(+), 43 deletions(-) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go index c4853dcb9ed7..0f3aae69e8c2 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns.go @@ -24,7 +24,7 @@ type ReplaceAllPatternsArguments[K any] struct { RegexPattern string Replacement ottl.StringGetter[K] Function ottl.Optional[ottl.FunctionGetter[K]] - ReplacementFormat ottl.Optional[string] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } func NewReplaceAllPatternsFactory[K any]() ottl.Factory[K] { @@ -41,7 +41,7 @@ func createReplaceAllPatternsFunction[K any](_ ottl.FunctionContext, oArgs ottl. return replaceAllPatterns(args.Target, args.Mode, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[string]) (ottl.ExprFunc[K], error) { +func replaceAllPatterns[K any](target ottl.PMapGetter[K], mode string, regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_all_patterns is not a valid pattern: %w", err) diff --git a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go index a6b3101a1691..896926a4046f 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_patterns_test.go @@ -30,6 +30,16 @@ func Test_replaceAllPatterns(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Map](), } + prefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=%s", nil + }, + } + invalidPrefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) target := &ottl.StandardPMapGetter[pcommon.Map]{ @@ -44,7 +54,7 @@ func Test_replaceAllPatterns(t *testing.T) { mode string pattern string replacement ottl.StringGetter[pcommon.Map] - replacementFormat ottl.Optional[string] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Map]] function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] want func(pcommon.Map) }{ @@ -58,7 +68,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hash(hello {universe}) world") @@ -139,11 +149,11 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementFormat: ottl.NewTestingOptional[string]("prefix=%s"), + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](prefix), function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "prefix=hash(hello {universe}) world") - expectedMap.PutStr("test2", "prefix=hash(hello {universe})") // replacement format is applied twice + expectedMap.PutStr("test2", "prefix=hash(hello {universe})") expectedMap.PutStr("test3", "goodbye world1 and world2") expectedMap.PutInt("test4", 1234) expectedMap.PutDouble("test5", 1234) @@ -160,7 +170,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementFormat: ottl.NewTestingOptional[string]("passwd="), + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](invalidPrefix), function: optionalArg, want: func(expectedMap pcommon.Map) { expectedMap.PutEmpty("test") @@ -177,7 +187,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "hello {universe}", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe} world") @@ -198,7 +208,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -219,7 +229,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "**** ", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello **** ") @@ -340,7 +350,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "foo", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -362,7 +372,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "nothing {matches}", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -384,7 +394,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test.", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -406,7 +416,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "world-$1", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -428,7 +438,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "test-$1", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") @@ -449,7 +459,7 @@ func Test_replaceAllPatterns(t *testing.T) { return "$$world-$1", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.Clear() @@ -494,7 +504,7 @@ func Test_replaceAllPatterns_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexpattern", replacement, function, replacementFormat) assert.NoError(t, err) @@ -516,7 +526,7 @@ func Test_replaceAllPatterns_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -546,7 +556,7 @@ func Test_replaceAllPatterns_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -568,7 +578,7 @@ func Test_replaceAllPatterns_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replaceAllPatterns[any](target, modeValue, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -590,7 +600,7 @@ func Test_replaceAllPatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidRegexPattern := "*" exprFunc, err := replaceAllPatterns[any](target, modeValue, invalidRegexPattern, replacement, function, replacementFormat) @@ -612,7 +622,7 @@ func Test_replaceAllPatterns_invalid_model(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidMode := "invalid" exprFunc, err := replaceAllPatterns[any](target, invalidMode, "regex", replacement, function, replacementFormat) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 2d43dc78ffc2..94a058229378 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -17,7 +17,7 @@ type ReplacePatternArguments[K any] struct { RegexPattern string Replacement ottl.StringGetter[K] Function ottl.Optional[ottl.FunctionGetter[K]] - ReplacementFormat ottl.Optional[string] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replacePatternFuncArgs[K any] struct { @@ -38,7 +38,7 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } -func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string, replacementFormat ottl.Optional[string]) (string, error) { +func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string, replacementFormat ottl.Optional[ottl.StringGetter[K]]) (string, error) { var updatedString string updatedString = originalValStr submatches := compiledPattern.FindAllStringSubmatchIndex(updatedString, -1) @@ -65,17 +65,21 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern } if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value formatString := replacementFormat.Get() - if !strings.Contains(formatString, "%s") { - return "", fmt.Errorf("replacementFormat must be format string with %%s") + formatStringVal, err := formatString.Get(ctx, tCtx) + if err != nil { + return "", err } - replacementValStr = fmt.Sprintf(replacementFormat.Get(), replacementValStr) + if strings.Count(formatStringVal, "%s") > 1 || strings.Count(formatStringVal, "%") < 1 { + return "", fmt.Errorf("replacementFormat must be format string containing a single %%s and no other format specifiers") + } + replacementValStr = fmt.Sprintf(formatStringVal, replacementValStr) } updatedString = strings.ReplaceAll(updatedString, fullMatch, replacementValStr) } return updatedString, nil } -func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[string]) (ottl.ExprFunc[K], error) { +func replacePattern[K any](target ottl.GetSetter[K], regexPattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { compiledPattern, err := regexp.Compile(regexPattern) if err != nil { return nil, fmt.Errorf("the regex pattern supplied to replace_pattern is not a valid pattern: %w", err) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go index 4393e1ef28ec..895eda7fe4e5 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern_test.go @@ -53,6 +53,16 @@ func Test_replacePattern(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Value](), } + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=%s", nil + }, + } + passwdSuffix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "%s (passwd)", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { @@ -69,7 +79,7 @@ func Test_replacePattern(t *testing.T) { target ottl.GetSetter[pcommon.Value] pattern string replacement ottl.StringGetter[pcommon.Value] - replacementFormat ottl.Optional[string] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Value]] function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] want func(pcommon.Value) }{ @@ -82,7 +92,7 @@ func Test_replacePattern(t *testing.T) { return "$1", nil }, }, - replacementFormat: ottl.NewTestingOptional[string]("passwd=%s"), + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix), function: optionalArg, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=hash(sensitivedtata)otherarg=notsensitive key1 key2") @@ -139,7 +149,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=***", nil }, }, - replacementFormat: ottl.NewTestingOptional[string]("%s (passwd)"), + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdSuffix), function: optionalArg, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application hash(passwd=***) (passwd) otherarg=notsensitive key1 key2") @@ -154,7 +164,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=*** ", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=*** otherarg=notsensitive key1 key2") @@ -169,7 +179,7 @@ func Test_replacePattern(t *testing.T) { return "shouldnotbeinoutput", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive key1 key2") @@ -184,7 +194,7 @@ func Test_replacePattern(t *testing.T) { return "**** ", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=sensitivedtata otherarg=notsensitive **** **** ") @@ -199,7 +209,7 @@ func Test_replacePattern(t *testing.T) { return "$1:$2", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd:sensitivedtata otherarg:notsensitive key1 key2") @@ -214,7 +224,7 @@ func Test_replacePattern(t *testing.T) { return "passwd=$$$$$$ ", nil }, }, - replacementFormat: ottl.Optional[string]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("application passwd=$$$ otherarg=notsensitive key1 key2") @@ -257,7 +267,7 @@ func Test_replacePattern_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -285,7 +295,7 @@ func Test_replacePattern_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -319,7 +329,7 @@ func Test_replacePattern_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replacePattern[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) @@ -346,7 +356,7 @@ func Test_replacePattern_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} exprFunc, err := replacePattern[any](target, `nomatch\=[^\s]*(\s?)`, replacement, function, replacementFormat) assert.NoError(t, err) @@ -373,7 +383,7 @@ func Test_replacePatterns_invalid_pattern(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} - replacementFormat := ottl.Optional[string]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} invalidRegexPattern := "*" _, err := replacePattern[any](target, invalidRegexPattern, replacement, function, replacementFormat) @@ -403,13 +413,18 @@ func Test_replacePattern_bad_format_string(t *testing.T) { }, Fact: StandardConverters[pcommon.Value]()["SHA256"], } - replacementFormat := ottl.NewTestingOptional[string]("passwd=") // This is not a valid format string + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=", nil + }, + } + replacementFormat := ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix) // This is not a valid format string function := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) exprFunc, err := replacePattern[pcommon.Value](target, `passwd\=[^\s]*`, replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) require.Error(t, err) - assert.ErrorContains(t, err, "replacementFormat must be format string with %s") + assert.ErrorContains(t, err, "replacementFormat must be format string containing a single %s") assert.Nil(t, result) } From f3f8844c4e0b614d34e967f404d194870c35fdf9 Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Fri, 9 Feb 2024 16:00:23 -0500 Subject: [PATCH 5/6] Add support for replacement format in replace_[all_]matches editors --- pkg/ottl/ottlfuncs/README.md | 2 + .../ottlfuncs/func_replace_all_matches.go | 18 +++++--- .../func_replace_all_matches_test.go | 45 ++++++++++++------- pkg/ottl/ottlfuncs/func_replace_match.go | 18 +++++--- pkg/ottl/ottlfuncs/func_replace_match_test.go | 43 +++++++++++------- pkg/ottl/ottlfuncs/func_replace_pattern.go | 28 +++++++----- 6 files changed, 99 insertions(+), 55 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 157a197e8e23..91d2a9be6a61 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -251,6 +251,7 @@ Examples: The `replace_all_matches` function replaces any matching string value with the replacement string. `target` is a path expression to a `pcommon.Map` type field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching string with the hash value of `replacement`. +`replacementFormat` is an optional string argument that specifies the format of the replacement. Each string value in `target` that matches `pattern` will get replaced with `replacement`. Non-string values are ignored. @@ -295,6 +296,7 @@ If using OTTL outside of collector configuration, `$` should not be escaped and The `replace_match` function allows replacing entire strings if they match a glob pattern. `target` is a path expression to a telemetry field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. +`replacementFormat` is an optional string argument that specifies the format of the replacement. If `target` matches `pattern` it will get replaced with `replacement`. diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches.go b/pkg/ottl/ottlfuncs/func_replace_all_matches.go index 597cf878a12c..1e4fc2874aa8 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches.go @@ -14,10 +14,11 @@ import ( ) type ReplaceAllMatchesArguments[K any] struct { - Target ottl.PMapGetter[K] - Pattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.PMapGetter[K] + Pattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replaceAllMatchesFuncArgs[K any] struct { @@ -35,10 +36,10 @@ func createReplaceAllMatchesFunction[K any](_ ottl.FunctionContext, oArgs ottl.A return nil, fmt.Errorf("ReplaceAllMatchesFactory args must be of type *ReplaceAllMatchesArguments[K]") } - return replaceAllMatches(args.Target, args.Pattern, args.Replacement, args.Function) + return replaceAllMatches(args.Target, args.Pattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) @@ -68,7 +69,10 @@ func replaceAllMatches[K any](target ottl.PMapGetter[K], pattern string, replace if !ok { return nil, fmt.Errorf("replacement value is not a string") } - replacementVal = replacementValStr + replacementVal, err = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if err != nil { + return nil, err + } } val.Range(func(key string, value pcommon.Value) bool { if glob.Match(value.Str()) { diff --git a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go index 3494aa0d4866..0ca6fc1dcdbb 100644 --- a/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_all_matches_test.go @@ -27,6 +27,11 @@ func Test_replaceAllMatches(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Map](), } + prefix := ottl.StandardStringGetter[pcommon.Map]{ + Getter: func(context.Context, pcommon.Map) (any, error) { + return "prefix=%s", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Map]](ottlValue) target := &ottl.StandardPMapGetter[pcommon.Map]{ @@ -36,12 +41,13 @@ func Test_replaceAllMatches(t *testing.T) { } tests := []struct { - name string - target ottl.PMapGetter[pcommon.Map] - pattern string - replacement ottl.StringGetter[pcommon.Map] - function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] - want func(pcommon.Map) + name string + target ottl.PMapGetter[pcommon.Map] + pattern string + replacement ottl.StringGetter[pcommon.Map] + function ottl.Optional[ottl.FunctionGetter[pcommon.Map]] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Map]] + want func(pcommon.Map) }{ { name: "replace only matches (with hash function)", @@ -52,10 +58,11 @@ func Test_replaceAllMatches(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + function: optionalArg, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Map]](prefix), want: func(expectedMap pcommon.Map) { - expectedMap.PutStr("test", "hash(hello {universe})") - expectedMap.PutStr("test2", "hash(hello {universe})") + expectedMap.PutStr("test", "prefix=hash(hello {universe})") + expectedMap.PutStr("test2", "prefix=hash(hello {universe})") expectedMap.PutStr("test3", "goodbye") }, }, @@ -68,7 +75,8 @@ func Test_replaceAllMatches(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello {universe}") expectedMap.PutStr("test2", "hello {universe}") @@ -84,7 +92,8 @@ func Test_replaceAllMatches(t *testing.T) { return "nothing {matches}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Map]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Map]]{}, want: func(expectedMap pcommon.Map) { expectedMap.PutStr("test", "hello world") expectedMap.PutStr("test2", "hello") @@ -97,7 +106,7 @@ func Test_replaceAllMatches(t *testing.T) { scenarioMap := pcommon.NewMap() input.CopyTo(scenarioMap) - exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceAllMatches(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioMap) @@ -125,8 +134,9 @@ func Test_replaceAllMatches_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "*", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, input) assert.Error(t, err) @@ -145,8 +155,9 @@ func Test_replaceAllMatches_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -174,8 +185,9 @@ func Test_replaceAllMatches_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -195,8 +207,9 @@ func Test_replaceAllMatches_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceAllMatches[any](target, "*", replacement, function) + exprFunc, err := replaceAllMatches[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) _, err = exprFunc(nil, nil) assert.Error(t, err) diff --git a/pkg/ottl/ottlfuncs/func_replace_match.go b/pkg/ottl/ottlfuncs/func_replace_match.go index 123320caa421..21cf825234bc 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match.go +++ b/pkg/ottl/ottlfuncs/func_replace_match.go @@ -13,10 +13,11 @@ import ( ) type ReplaceMatchArguments[K any] struct { - Target ottl.GetSetter[K] - Pattern string - Replacement ottl.StringGetter[K] - Function ottl.Optional[ottl.FunctionGetter[K]] + Target ottl.GetSetter[K] + Pattern string + Replacement ottl.StringGetter[K] + Function ottl.Optional[ottl.FunctionGetter[K]] + ReplacementFormat ottl.Optional[ottl.StringGetter[K]] } type replaceMatchFuncArgs[K any] struct { @@ -34,10 +35,10 @@ func createReplaceMatchFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argume return nil, fmt.Errorf("ReplaceMatchFactory args must be of type *ReplaceMatchArguments[K]") } - return replaceMatch(args.Target, args.Pattern, args.Replacement, args.Function) + return replaceMatch(args.Target, args.Pattern, args.Replacement, args.Function, args.ReplacementFormat) } -func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]]) (ottl.ExprFunc[K], error) { +func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement ottl.StringGetter[K], fn ottl.Optional[ottl.FunctionGetter[K]], replacementFormat ottl.Optional[ottl.StringGetter[K]]) (ottl.ExprFunc[K], error) { glob, err := glob.Compile(pattern) if err != nil { return nil, fmt.Errorf("the pattern supplied to replace_match is not a valid pattern: %w", err) @@ -67,7 +68,10 @@ func replaceMatch[K any](target ottl.GetSetter[K], pattern string, replacement o if !ok { return nil, fmt.Errorf("replacement value is not a string") } - replacementVal = replacementValStr + replacementVal, err = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if err != nil { + return nil, err + } } if err != nil { return nil, err diff --git a/pkg/ottl/ottlfuncs/func_replace_match_test.go b/pkg/ottl/ottlfuncs/func_replace_match_test.go index a61e28560f90..d390d4e3165c 100644 --- a/pkg/ottl/ottlfuncs/func_replace_match_test.go +++ b/pkg/ottl/ottlfuncs/func_replace_match_test.go @@ -23,6 +23,11 @@ func Test_replaceMatch(t *testing.T) { }, Fact: optionalFnTestFactory[pcommon.Value](), } + passwdPrefix := ottl.StandardStringGetter[pcommon.Value]{ + Getter: func(context.Context, pcommon.Value) (any, error) { + return "passwd=%s", nil + }, + } optionalArg := ottl.NewTestingOptional[ottl.FunctionGetter[pcommon.Value]](ottlValue) target := &ottl.StandardGetSetter[pcommon.Value]{ Getter: func(ctx context.Context, tCtx pcommon.Value) (any, error) { @@ -35,12 +40,13 @@ func Test_replaceMatch(t *testing.T) { } tests := []struct { - name string - target ottl.GetSetter[pcommon.Value] - pattern string - replacement ottl.StringGetter[pcommon.Value] - function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] - want func(pcommon.Value) + name string + target ottl.GetSetter[pcommon.Value] + pattern string + replacement ottl.StringGetter[pcommon.Value] + function ottl.Optional[ottl.FunctionGetter[pcommon.Value]] + replacementFormat ottl.Optional[ottl.StringGetter[pcommon.Value]] + want func(pcommon.Value) }{ { name: "replace match (with hash function)", @@ -51,9 +57,10 @@ func Test_replaceMatch(t *testing.T) { return "hello {universe}", nil }, }, - function: optionalArg, + function: optionalArg, + replacementFormat: ottl.NewTestingOptional[ottl.StringGetter[pcommon.Value]](passwdPrefix), want: func(expectedValue pcommon.Value) { - expectedValue.SetStr("hash(hello {universe})") + expectedValue.SetStr("passwd=hash(hello {universe})") }, }, { @@ -65,7 +72,8 @@ func Test_replaceMatch(t *testing.T) { return "hello {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello {universe}") }, @@ -79,7 +87,8 @@ func Test_replaceMatch(t *testing.T) { return "goodbye {universe}", nil }, }, - function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + function: ottl.Optional[ottl.FunctionGetter[pcommon.Value]]{}, + replacementFormat: ottl.Optional[ottl.StringGetter[pcommon.Value]]{}, want: func(expectedValue pcommon.Value) { expectedValue.SetStr("hello world") }, @@ -89,7 +98,7 @@ func Test_replaceMatch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { scenarioValue := pcommon.NewValueStr(input.Str()) - exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement, tt.function) + exprFunc, err := replaceMatch(tt.target, tt.pattern, tt.replacement, tt.function, tt.replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, scenarioValue) assert.NoError(t, err) @@ -120,8 +129,9 @@ func Test_replaceMatch_bad_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "*", replacement, function) + exprFunc, err := replaceMatch[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -148,8 +158,9 @@ func Test_replaceMatch_bad_function_input(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "regexp", replacement, function) + exprFunc, err := replaceMatch[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -181,8 +192,9 @@ func Test_replaceMatch_bad_function_result(t *testing.T) { Fact: StandardConverters[any]()["IsString"], } function := ottl.NewTestingOptional[ottl.FunctionGetter[any]](ottlValue) + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "regexp", replacement, function) + exprFunc, err := replaceMatch[any](target, "regexp", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, input) @@ -207,8 +219,9 @@ func Test_replaceMatch_get_nil(t *testing.T) { }, } function := ottl.Optional[ottl.FunctionGetter[any]]{} + replacementFormat := ottl.Optional[ottl.StringGetter[any]]{} - exprFunc, err := replaceMatch[any](target, "*", replacement, function) + exprFunc, err := replaceMatch[any](target, "*", replacement, function, replacementFormat) assert.NoError(t, err) result, err := exprFunc(nil, nil) diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 94a058229378..8ec05d940751 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -38,6 +38,21 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } +func applyReplaceFormat[K any](ctx context.Context, tCtx K, replacementFormat ottl.Optional[ottl.StringGetter[K]], replacementVal string) (string, error) { + if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value + formatString := replacementFormat.Get() + formatStringVal, errFmt := formatString.Get(ctx, tCtx) + if errFmt != nil { + return "", errFmt + } + if strings.Count(formatStringVal, "%s") > 1 || strings.Count(formatStringVal, "%") < 1 { + return "", fmt.Errorf("replacementFormat must be format string containing a single %%s and no other format specifiers") + } + replacementVal = fmt.Sprintf(formatStringVal, replacementVal) + } + return replacementVal, nil +} + func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern *regexp.Regexp, fn ottl.Optional[ottl.FunctionGetter[K]], originalValStr string, replacementVal string, replacementFormat ottl.Optional[ottl.StringGetter[K]]) (string, error) { var updatedString string updatedString = originalValStr @@ -63,16 +78,9 @@ func applyOptReplaceFunction[K any](ctx context.Context, tCtx K, compiledPattern if !ok { return "", fmt.Errorf("the replacement value must be a string") } - if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value - formatString := replacementFormat.Get() - formatStringVal, err := formatString.Get(ctx, tCtx) - if err != nil { - return "", err - } - if strings.Count(formatStringVal, "%s") > 1 || strings.Count(formatStringVal, "%") < 1 { - return "", fmt.Errorf("replacementFormat must be format string containing a single %%s and no other format specifiers") - } - replacementValStr = fmt.Sprintf(formatStringVal, replacementValStr) + replacementValStr, errNew = applyReplaceFormat(ctx, tCtx, replacementFormat, replacementValStr) + if errNew != nil { + return "", errNew } updatedString = strings.ReplaceAll(updatedString, fullMatch, replacementValStr) } From b72a5256e45be1b72ddc829a632eeff8c57c0a77 Mon Sep 17 00:00:00 2001 From: Raj Nishtala Date: Fri, 9 Feb 2024 18:41:21 -0500 Subject: [PATCH 6/6] Changed the validation for the replacement format string --- pkg/ottl/ottlfuncs/README.md | 12 ++++++------ pkg/ottl/ottlfuncs/func_replace_pattern.go | 12 +++++++++++- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 91d2a9be6a61..e97efc699335 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -251,14 +251,14 @@ Examples: The `replace_all_matches` function replaces any matching string value with the replacement string. `target` is a path expression to a `pcommon.Map` type field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching string with the hash value of `replacement`. -`replacementFormat` is an optional string argument that specifies the format of the replacement. +`replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. Each string value in `target` that matches `pattern` will get replaced with `replacement`. Non-string values are ignored. Examples: - `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}")` -- `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` +- `replace_all_matches(attributes, "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256, "/user/%s")` ### replace_all_patterns @@ -273,7 +273,7 @@ The `replace_all_patterns` function replaces any segments in a string value or k If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces any matching regex pattern with the hash value of `replacement`. @@ -296,7 +296,7 @@ If using OTTL outside of collector configuration, `$` should not be escaped and The `replace_match` function allows replacing entire strings if they match a glob pattern. `target` is a path expression to a telemetry field. `pattern` is a string following [filepath.Match syntax](https://pkg.go.dev/path/filepath#Match). `replacement` is either a path expression to a string telemetry field or a literal string. -`replacementFormat` is an optional string argument that specifies the format of the replacement. +`replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported. If `target` matches `pattern` it will get replaced with `replacement`. @@ -305,7 +305,7 @@ The `function` is an optional argument that can take in any Converter that accep Examples: - `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}")` -- `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256)` +- `replace_match(attributes["http.target"], "/user/*/list/*", "/user/{userId}/list/{listId}", SHA256, "/user/%s")` ### replace_pattern @@ -319,7 +319,7 @@ The `replace_pattern` function allows replacing all string sections that match a If one or more sections of `target` match `regex` they will get replaced with `replacement`. -The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. +The `replacement` string can refer to matched groups using [regexp.Expand syntax](https://pkg.go.dev/regexp#Regexp.Expand). `replacementFormat` is an optional string argument that specifies the format of the replacement. It must contain exactly one `%s` format specifier as shown in the example below. No other format specifiers are supported The `function` is an optional argument that can take in any Converter that accepts a (`replacement`) string and returns a string. An example is a hash function that replaces a matching regex pattern with the hash value of `replacement`. diff --git a/pkg/ottl/ottlfuncs/func_replace_pattern.go b/pkg/ottl/ottlfuncs/func_replace_pattern.go index 8ec05d940751..b77d3463d19b 100644 --- a/pkg/ottl/ottlfuncs/func_replace_pattern.go +++ b/pkg/ottl/ottlfuncs/func_replace_pattern.go @@ -38,6 +38,16 @@ func createReplacePatternFunction[K any](_ ottl.FunctionContext, oArgs ottl.Argu return replacePattern(args.Target, args.RegexPattern, args.Replacement, args.Function, args.ReplacementFormat) } +func validFormatString(formatString string) bool { + // Check for exactly one %s and no other invalid format specifiers + validPattern := `^(.*?%s.*?)$` + validRegex := regexp.MustCompile(validPattern) + invalidPattern := `%[^s]` + invalidRegex := regexp.MustCompile(invalidPattern) + + return validRegex.MatchString(formatString) && !invalidRegex.MatchString(formatString) +} + func applyReplaceFormat[K any](ctx context.Context, tCtx K, replacementFormat ottl.Optional[ottl.StringGetter[K]], replacementVal string) (string, error) { if !replacementFormat.IsEmpty() { // If replacementFormat is not empty, add it to the replacement value formatString := replacementFormat.Get() @@ -45,7 +55,7 @@ func applyReplaceFormat[K any](ctx context.Context, tCtx K, replacementFormat ot if errFmt != nil { return "", errFmt } - if strings.Count(formatStringVal, "%s") > 1 || strings.Count(formatStringVal, "%") < 1 { + if !validFormatString(formatStringVal) { return "", fmt.Errorf("replacementFormat must be format string containing a single %%s and no other format specifiers") } replacementVal = fmt.Sprintf(formatStringVal, replacementVal)