From ea00d3bbce8bd81cc4cda2fd951ef549bb40ac8e Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 16 Nov 2022 09:27:36 -0700 Subject: [PATCH 1/4] Add ParseJSONIntoMap function --- pkg/ottl/ottlfuncs/README.md | 49 +++++ .../ottlfuncs/func_parse_json_into_map.go | 91 ++++++++ .../func_parse_json_into_map_test.go | 194 ++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 pkg/ottl/ottlfuncs/func_parse_json_into_map.go create mode 100644 pkg/ottl/ottlfuncs/func_parse_json_into_map_test.go diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index 9ce7ac7092048..ebcfa665bc9e6 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -16,6 +16,7 @@ Functions - [delete_matching_keys](#delete_matching_keys) - [keep_keys](#keep_keys) - [limit](#limit) +- [parse_json_into_map](#parse_json_into_map) - [replace_all_matches](#replace_all_matches) - [replace_all_patterns](#replace_all_patterns) - [replace_match](#replace_match) @@ -220,6 +221,54 @@ Examples: - `limit(resource.attributes, 50, ["http.host", "http.method"])` + + + + + + + + +## parse_json_into_map + +`parse_json_into_map(target, value)` + +The `parse_json_into_map` function unmarshals the value string as json and updates/inserts the json object's root fields into the target map. + +`target` is a path expression to a `pdata.Map` type field. `value` is a string or a path expression or function that returns a string. + +`value` is unmarshalled using [jsoniter](https://github.com/json-iterator/go). Each JSON type is converted into a `pdata.Value` using the following map: + +``` +JSON boolean -> bool +JSON number -> float64 +JSON string -> string +JSON null -> nil +JSON arrays -> pdata.SliceValue +JSON objects -> string +``` + +The OTTL contexts don't know how to set maps, so string is used for JSON objects instead of map [#16337](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/16337). +Using strings for now allows follow-up statements to parse any nested objects. + +Examples: + +- `parse_json_to_map(attributes, "{\"attr\":true}")` +- `parse_json_to_map(attributes, attributes["kubernetes"])` +- `parse_json_to_map(attributes, body)` +- `parse_json_to_map(attributes, SomeFunctionThatReturnsJSON())` + + + + + + + + + + + + ## replace_all_matches `replace_all_matches(target, pattern, replacement)` diff --git a/pkg/ottl/ottlfuncs/func_parse_json_into_map.go b/pkg/ottl/ottlfuncs/func_parse_json_into_map.go new file mode 100644 index 0000000000000..9ff4c9c86c400 --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_parse_json_into_map.go @@ -0,0 +1,91 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ottlfuncs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl/ottlfuncs" + +import ( + "context" + + jsoniter "github.com/json-iterator/go" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func ParseJSONIntoMap[K any](target ottl.GetSetter[K], value ottl.Getter[K]) (ottl.ExprFunc[K], error) { + return func(ctx context.Context, tCtx K) (interface{}, error) { + targetVal, err := target.Get(ctx, tCtx) + if err != nil { + return nil, err + } + if attrs, ok := targetVal.(pcommon.Map); ok { + val, err := value.Get(ctx, tCtx) + if err != nil { + return nil, err + } + if valStr, ok := val.(string); ok { + var parsedValue map[string]interface{} + err = jsoniter.UnmarshalFromString(valStr, &parsedValue) + if err != nil { + return nil, err + } + for k, v := range parsedValue { + attrVal := pcommon.NewValueEmpty() + err = setValue(attrVal, v) + if err != nil { + return nil, err + } + attrVal.CopyTo(attrs.PutEmpty(k)) + } + } + } + return nil, nil + }, nil +} + +func setValue(value pcommon.Value, val interface{}) error { + switch v := val.(type) { + case string: + value.SetStr(v) + case bool: + value.SetBool(v) + case float64: + value.SetDouble(v) + case nil: + case []interface{}: + emptySlice := value.SetEmptySlice() + err := setSlice(emptySlice, v) + if err != nil { + return err + } + case map[string]interface{}: + mapStr, err := jsoniter.MarshalToString(val) + if err != nil { + return err + } + value.SetStr(mapStr) + } + return nil +} + +func setSlice(slice pcommon.Slice, value []interface{}) error { + for _, item := range value { + emptyValue := slice.AppendEmpty() + err := setValue(emptyValue, item) + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/ottl/ottlfuncs/func_parse_json_into_map_test.go b/pkg/ottl/ottlfuncs/func_parse_json_into_map_test.go new file mode 100644 index 0000000000000..72775316a1caf --- /dev/null +++ b/pkg/ottl/ottlfuncs/func_parse_json_into_map_test.go @@ -0,0 +1,194 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ottlfuncs + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl" +) + +func Test_ParseJSONIntoMap(t *testing.T) { + input := pcommon.NewMap() + input.PutStr("existing", "attr") + + target := &ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return tCtx, nil + }, + Setter: func(ctx context.Context, tCtx pcommon.Map, val interface{}) error { + val.(pcommon.Map).CopyTo(tCtx) + return nil + }, + } + + tests := []struct { + name string + target ottl.GetSetter[pcommon.Map] + value ottl.Getter[pcommon.Map] + want func(pcommon.Map) + }{ + { + name: "handle string", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":"string value"}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", "string value") + }, + }, + { + name: "handle bool", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":true}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutBool("test", true) + }, + }, + { + name: "handle int", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":1}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutDouble("test", 1) + }, + }, + { + name: "handle float", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":1.1}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutDouble("test", 1.1) + }, + }, + { + name: "handle nil", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":null}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutEmpty("test") + }, + }, + { + name: "handle array", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":["string","value"]}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + emptySlice := expectedMap.PutEmptySlice("test") + emptySlice.AppendEmpty().SetStr("string") + emptySlice.AppendEmpty().SetStr("value") + }, + }, + { + name: "handle nested object", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test":{"nested":"true"}}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test", `{"nested":"true"}`) + }, + }, + { + name: "updates existing", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"existing":"pass"}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("existing", "pass") + }, + }, + { + name: "complex", + target: target, + value: ottl.StandardGetSetter[pcommon.Map]{ + Getter: func(ctx context.Context, tCtx pcommon.Map) (interface{}, error) { + return `{"test1":{"nested":"true"},"test2":"string","test3":1,"test4":1.1,"test5":["string", 1, [2, 3],{"nested":true}],"test6":null}`, nil + }, + }, + want: func(expectedMap pcommon.Map) { + expectedMap.PutStr("test1", `{"nested":"true"}`) + expectedMap.PutStr("test2", "string") + expectedMap.PutDouble("test3", 1) + expectedMap.PutDouble("test4", 1.1) + slice := expectedMap.PutEmptySlice("test5") + slice.AppendEmpty().SetStr("string") + slice.AppendEmpty().SetDouble(1) + nestedSlice := slice.AppendEmpty().SetEmptySlice() + nestedSlice.AppendEmpty().SetDouble(2) + nestedSlice.AppendEmpty().SetDouble(3) + slice.AppendEmpty().SetStr(`{"nested":true}`) + expectedMap.PutEmpty("test6") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scenarioMap := pcommon.NewMap() + input.CopyTo(scenarioMap) + + exprFunc, err := ParseJSONIntoMap(tt.target, tt.value) + assert.NoError(t, err) + + result, err := exprFunc(context.Background(), scenarioMap) + assert.NoError(t, err) + assert.Nil(t, result) + + expected := pcommon.NewMap() + input.CopyTo(expected) + tt.want(expected) + + assert.Equal(t, expected.Len(), scenarioMap.Len()) + expected.Range(func(k string, v pcommon.Value) bool { + ev, _ := expected.Get(k) + av, _ := scenarioMap.Get(k) + assert.Equal(t, ev, av) + return true + }) + }) + } +} From c04074f875ec2343037c62ec80cd1ec7413646bd Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:28:18 -0700 Subject: [PATCH 2/4] Fix formatting --- pkg/ottl/ottlfuncs/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/ottl/ottlfuncs/README.md b/pkg/ottl/ottlfuncs/README.md index ebcfa665bc9e6..d64dc3bb47b0d 100644 --- a/pkg/ottl/ottlfuncs/README.md +++ b/pkg/ottl/ottlfuncs/README.md @@ -221,14 +221,6 @@ Examples: - `limit(resource.attributes, 50, ["http.host", "http.method"])` - - - - - - - - ## parse_json_into_map `parse_json_into_map(target, value)` @@ -258,17 +250,6 @@ Examples: - `parse_json_to_map(attributes, body)` - `parse_json_to_map(attributes, SomeFunctionThatReturnsJSON())` - - - - - - - - - - - ## replace_all_matches `replace_all_matches(target, pattern, replacement)` From 4aab23cb52b4aa4510e24464cb7780ba97fa5663 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:32:22 -0700 Subject: [PATCH 3/4] Add changelog entry --- .chloggen/ottl-parse-json.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 .chloggen/ottl-parse-json.yaml diff --git a/.chloggen/ottl-parse-json.yaml b/.chloggen/ottl-parse-json.yaml new file mode 100755 index 0000000000000..bf17e40d2bb71 --- /dev/null +++ b/.chloggen/ottl-parse-json.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/ottl + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add new `ParseJSONIntoMap` function to convert json strings into attributes. + +# One or more tracking issues related to the change +issues: [16340] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: From 1a2a16c0f17e0c615801f71097daad2507284599 Mon Sep 17 00:00:00 2001 From: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com> Date: Wed, 16 Nov 2022 15:46:24 -0700 Subject: [PATCH 4/4] run go mod tidy --- pkg/ottl/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ottl/go.mod b/pkg/ottl/go.mod index 4aa6458825854..dc65f1d7936a0 100644 --- a/pkg/ottl/go.mod +++ b/pkg/ottl/go.mod @@ -6,6 +6,7 @@ require ( github.com/alecthomas/participle/v2 v2.0.0-beta.5 github.com/gobwas/glob v0.2.3 github.com/iancoleman/strcase v0.2.0 + github.com/json-iterator/go v1.1.12 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.8.1 go.opentelemetry.io/collector v0.64.2-0.20221115155901-1550938c18fd @@ -20,7 +21,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect github.com/knadh/koanf v1.4.4 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect