From 6b21b6b1b43f3b485236828ef6ffd5936440db93 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 27 Nov 2019 16:25:07 -0500 Subject: [PATCH 01/13] [WIP] Second draft for multiples output Its a bit messy but I want to get the fleet branch in POC stats ASAP. --- x-pack/agent/pkg/agent/program/program.go | 122 ++++++++++++++++++ .../agent/pkg/agent/program/program_test.go | 101 +++++++++++++++ x-pack/agent/pkg/agent/transpiler/ast.go | 15 +++ x-pack/agent/pkg/agent/transpiler/rules.go | 2 +- 4 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 x-pack/agent/pkg/agent/program/program_test.go diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index ad4443abb2c0..35dd2fddebea 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -5,8 +5,11 @@ package program import ( + "fmt" "strings" + "github.com/pkg/errors" + "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" "github.com/elastic/beats/x-pack/agent/pkg/boolexp" ) @@ -90,3 +93,122 @@ func KnownProgramNames() []string { } return names } + +func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { + const ( + outputsKey = "outputs" + outputKey = "output" + streamsKey = "streams" + typeKey = "type" + ) + + if _, found := transpiler.Select(single, outputsKey); !found { + return nil, errors.New("invalid configuration missing outputs configuration") + } + + // Normalize using an intermediate map. + normMap, err := single.Map() + if err != nil { + return nil, errors.Wrap(err, "could not read configuration") + } + + // Recreates multiple configuration grouped by the name of the outputs. + // Each configuration will be started into his own operator with the same name as the output. + grouped := make(map[string]map[string]interface{}) + + m, ok := normMap[outputsKey] + if !ok { + return nil, errors.New("fail to received a list of configured outputs") + } + + out := m.(map[string]interface{}) + + for k, v := range out { + outputsOptions, ok := v.(map[string]interface{}) + if !ok { + return nil, errors.New("invalid type for output configuration block") + } + + t, ok := outputsOptions[typeKey] + if !ok { + return nil, fmt.Errorf("missing output type named output %s", k) + } + + n, ok := t.(string) + if !ok { + return nil, fmt.Errorf("invalid type received %T and expecting a string", t) + } + + grouped[k] = map[string]interface{}{ + outputKey: map[string]interface{}{n: v}, + streamsKey: make([]map[string]interface{}, 0), + } + } + + s, ok := normMap[streamsKey] + if !ok { + return nil, errors.New("no streams are configured") + } + + list, ok := s.([]interface{}) + if !ok { + return nil, errors.New("fail to receive a list of configured streams") + } + + for _, item := range list { + stream, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf( + "invalid type for stream expecting a map of options and received %T", + item, + ) + } + targetName := findNamedOutput(stream) + + // Do we have configuration for that specific outputs if not we fail to load the configuration. + config, ok := grouped[targetName] + if !ok { + return nil, fmt.Errorf("unknown configuration output with name %s", targetName) + } + + streams := config[streamsKey].([]map[string]interface{}) + streams = append(streams, stream) + + config[streamsKey] = streams + grouped[targetName] = config + } + + transpiled := make(map[string]*transpiler.AST) + + for name, group := range grouped { + ast, err := transpiler.NewAST(group) + if err != nil { + return nil, errors.Wrapf(err, "fail to generate configuration for output name %s", name) + } + + transpiled[name] = ast + } + + return transpiled, nil +} + +func findNamedOutput(m map[string]interface{}) string { + const ( + defaultOutputName = "default" + outputKey = "output" + useOutputKey = "use_output" + ) + + output, ok := m[outputKey] + if !ok { + return defaultOutputName + } + + o := output.(map[string]interface{}) + + name, ok := o[useOutputKey] + if !ok { + return defaultOutputName + } + return name.(string) +} diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go new file mode 100644 index 000000000000..6449f69a6528 --- /dev/null +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -0,0 +1,101 @@ +package program + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" +) + +func TestGroupBy(t *testing.T) { + t.Run("default and named output", func(t *testing.T) { + sConfig := map[string]interface{}{ + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + "infosec1": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/path1.log", + }, + map[string]interface{}{ + "type": "metrics/system", + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/path2.log", + "output": map[string]interface{}{ + "user_output": "infosec1", + }, + }, + }, + } + + ast, err := transpiler.NewAST(sConfig) + require.NoError(t, err) + + grouped, err := groupBy(ast) + require.NoError(t, err) + require.Equal(t, 2, len(grouped)) + + c1, _ := transpiler.NewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + }, + map[string]interface{}{ + "type": "metrics/system", + }, + }, + }) + + c2, _ := transpiler.NewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + }, + }, + }) + + defaultConfig, ok := grouped["default"] + require.True(t, ok) + fmt.Println("initial") + fmt.Println(c1) + fmt.Println("compareWith") + fmt.Println(defaultConfig) + require.Equal(t, c1.Hash(), defaultConfig.Hash()) + + infosec1Config, ok := grouped["infosec1"] + require.True(t, ok) + require.Equal(t, c2.Hash(), infosec1Config.Hash()) + }) +} diff --git a/x-pack/agent/pkg/agent/transpiler/ast.go b/x-pack/agent/pkg/agent/transpiler/ast.go index 4b1f29036f1d..929776eb3474 100644 --- a/x-pack/agent/pkg/agent/transpiler/ast.go +++ b/x-pack/agent/pkg/agent/transpiler/ast.go @@ -88,6 +88,11 @@ func (d *Dict) String() string { return sb.String() } +// Len returns the number of items in the dictionnary. +func (d *Dict) Len() int { + return len(d.value) +} + // Value returns the value of dict which is a slice of node. func (d *Dict) Value() interface{} { return d.value @@ -155,6 +160,11 @@ func (k *Key) Hash() []byte { return h.Sum(nil) } +// Name return the name of the key. +func (k *Key) Name() string { + return k.name +} + // List represents a slice in our Tree. type List struct { value []Node @@ -173,6 +183,11 @@ func (l *List) String() string { return sb.String() } +// Len returns the number of items in the list. +func (l *List) Len() int { + return len(l.value) +} + // Hash compute a sha256 hash of the current node and recursively call any children. func (l *List) Hash() []byte { h := sha256.New() diff --git a/x-pack/agent/pkg/agent/transpiler/rules.go b/x-pack/agent/pkg/agent/transpiler/rules.go index f2a3f7457b36..198ca5f914d7 100644 --- a/x-pack/agent/pkg/agent/transpiler/rules.go +++ b/x-pack/agent/pkg/agent/transpiler/rules.go @@ -62,7 +62,7 @@ func (r *RuleList) MarshalYAML() (interface{}, error) { case *FilterValuesWithRegexpRule: name = "filter_values_with_regexp" default: - return nil, fmt.Errorf("unkown rule of type %T", rule) + return nil, fmt.Errorf("unknown rule of type %T", rule) } subdoc := map[string]Rule{ From 5a72c3e41808b1814136289a838b87897c8381f5 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Wed, 27 Nov 2019 16:27:05 -0500 Subject: [PATCH 02/13] oups remove that --- x-pack/agent/pkg/agent/program/program_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 6449f69a6528..9b6cce55fc7b 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -1,7 +1,6 @@ package program import ( - "fmt" "testing" "github.com/stretchr/testify/require" @@ -88,10 +87,6 @@ func TestGroupBy(t *testing.T) { defaultConfig, ok := grouped["default"] require.True(t, ok) - fmt.Println("initial") - fmt.Println(c1) - fmt.Println("compareWith") - fmt.Println(defaultConfig) require.Equal(t, c1.Hash(), defaultConfig.Hash()) infosec1Config, ok := grouped["infosec1"] From ea8fb47ba58b02c0900fc476d94db88651f0c0db Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 10:25:32 -0500 Subject: [PATCH 03/13] adding more tests --- x-pack/agent/pkg/agent/program/program.go | 7 + .../agent/pkg/agent/program/program_test.go | 258 +++++++++++++++++- 2 files changed, 261 insertions(+), 4 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index 35dd2fddebea..4f9fc0f1afa6 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -139,6 +139,8 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { return nil, fmt.Errorf("invalid type received %T and expecting a string", t) } + delete(outputsOptions, typeKey) + grouped[k] = map[string]interface{}{ outputKey: map[string]interface{}{n: v}, streamsKey: make([]map[string]interface{}, 0), @@ -181,6 +183,10 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { transpiled := make(map[string]*transpiler.AST) for name, group := range grouped { + if len(group[streamsKey].([]map[string]interface{})) == 0 { + continue + } + ast, err := transpiler.NewAST(group) if err != nil { return nil, errors.Wrapf(err, "fail to generate configuration for output name %s", name) @@ -210,5 +216,6 @@ func findNamedOutput(m map[string]interface{}) string { if !ok { return defaultOutputName } + return name.(string) } diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 9b6cce55fc7b..67c87bf46681 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -8,9 +8,251 @@ import ( "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" ) +func placeHolder(t *testing.T) {} + func TestGroupBy(t *testing.T) { + t.Run("only named output", func(t *testing.T) { + sConfig := map[string]interface{}{ + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "localhost", + }, + }, + "outputs": map[string]interface{}{ + "special": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + "infosec1": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "metrics/system", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", + }, + }, + }, + } + + ast, err := transpiler.NewAST(sConfig) + require.NoError(t, err) + + grouped, err := groupBy(ast) + require.NoError(t, err) + require.Equal(t, 2, len(grouped)) + + c1 := transpiler.MustNewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "metrics/system", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + }, + }) + + c2, _ := transpiler.NewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", + }, + }, + }, + }) + + defaultConfig, ok := grouped["special"] + require.True(t, ok) + require.Equal(t, c1.Hash(), defaultConfig.Hash()) + + infosec1Config, ok := grouped["infosec1"] + + require.True(t, ok) + require.Equal(t, c2.Hash(), infosec1Config.Hash()) + }) + + t.Run("outputs with monitoring options", placeHolder) + + t.Run("fail when the referenced named output doesn't exist", func(t *testing.T) { + sConfig := map[string]interface{}{ + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "localhost", + }, + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + "infosec1": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + }, + map[string]interface{}{ + "type": "metrics/system", + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "donotexist", + "pipeline": "custompipeline", + "index_name": "myindex", + }, + }, + }, + } + + ast, err := transpiler.NewAST(sConfig) + require.NoError(t, err) + + _, err = groupBy(ast) + require.Error(t, err) + }) + + t.Run("only default output", func(t *testing.T) { + sConfig := map[string]interface{}{ + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "localhost", + }, + }, + "outputs": map[string]interface{}{ + "default": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + "infosec1": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + }, + map[string]interface{}{ + "type": "metrics/system", + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + }, + }, + } + + ast, err := transpiler.NewAST(sConfig) + require.NoError(t, err) + + grouped, err := groupBy(ast) + require.NoError(t, err) + require.Equal(t, 1, len(grouped)) + + c1 := transpiler.MustNewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + }, + map[string]interface{}{ + "type": "metrics/system", + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + }, + }, + }) + + defaultConfig, ok := grouped["default"] + require.True(t, ok) + require.Equal(t, c1.Hash(), defaultConfig.Hash()) + + _, ok = grouped["infosec1"] + + require.False(t, ok) + }) + t.Run("default and named output", func(t *testing.T) { sConfig := map[string]interface{}{ + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "localhost", + }, + }, "outputs": map[string]interface{}{ "default": map[string]interface{}{ "type": "elasticsearch", @@ -28,16 +270,18 @@ func TestGroupBy(t *testing.T) { "streams": []map[string]interface{}{ map[string]interface{}{ "type": "log", - "path": "/var/log/path1.log", + "path": "/var/log/hello.log", }, map[string]interface{}{ "type": "metrics/system", }, map[string]interface{}{ "type": "log", - "path": "/var/log/path2.log", + "path": "/var/log/infosec.log", "output": map[string]interface{}{ - "user_output": "infosec1", + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", }, }, }, @@ -50,7 +294,7 @@ func TestGroupBy(t *testing.T) { require.NoError(t, err) require.Equal(t, 2, len(grouped)) - c1, _ := transpiler.NewAST(map[string]interface{}{ + c1 := transpiler.MustNewAST(map[string]interface{}{ "output": map[string]interface{}{ "elasticsearch": map[string]interface{}{ "hosts": "xxx", @@ -81,6 +325,11 @@ func TestGroupBy(t *testing.T) { map[string]interface{}{ "type": "log", "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", + }, }, }, }) @@ -90,6 +339,7 @@ func TestGroupBy(t *testing.T) { require.Equal(t, c1.Hash(), defaultConfig.Hash()) infosec1Config, ok := grouped["infosec1"] + require.True(t, ok) require.Equal(t, c2.Hash(), infosec1Config.Hash()) }) From 5c70be44dd1bfc0954305d7d968d97d35e1368e9 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 11:08:04 -0500 Subject: [PATCH 04/13] fix the issue with configuration --- x-pack/agent/pkg/agent/program/program.go | 29 +++- .../agent/pkg/agent/program/program_test.go | 140 ++++++++++++++++-- 2 files changed, 148 insertions(+), 21 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index 4f9fc0f1afa6..5568f80065b4 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -141,10 +141,13 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { delete(outputsOptions, typeKey) - grouped[k] = map[string]interface{}{ - outputKey: map[string]interface{}{n: v}, - streamsKey: make([]map[string]interface{}, 0), - } + // Propagate global configuration to each individual configuration. + clone := cloneMap(normMap) + delete(clone, outputsKey) + clone[outputKey] = map[string]interface{}{n: v} + clone[streamsKey] = make([]map[string]interface{}, 0) + + grouped[k] = clone } s, ok := normMap[streamsKey] @@ -165,7 +168,7 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { item, ) } - targetName := findNamedOutput(stream) + targetName := findOutputName(stream) // Do we have configuration for that specific outputs if not we fail to load the configuration. config, ok := grouped[targetName] @@ -198,7 +201,7 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { return transpiled, nil } -func findNamedOutput(m map[string]interface{}) string { +func findOutputName(m map[string]interface{}) string { const ( defaultOutputName = "default" outputKey = "output" @@ -219,3 +222,17 @@ func findNamedOutput(m map[string]interface{}) string { return name.(string) } + +func cloneMap(m map[string]interface{}) map[string]interface{} { + newMap := make(map[string]interface{}) + for k, v := range m { + sV, ok := v.(map[string]interface{}) + if ok { + newMap[k] = cloneMap(sV) + continue + } + newMap[k] = v + } + + return newMap +} diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 67c87bf46681..72cb93f49cd1 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -8,16 +8,122 @@ import ( "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" ) -func placeHolder(t *testing.T) {} - func TestGroupBy(t *testing.T) { t.Run("only named output", func(t *testing.T) { + sConfig := map[string]interface{}{ + "outputs": map[string]interface{}{ + "special": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + "infosec1": map[string]interface{}{ + "type": "elasticsearch", + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "metrics/system", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", + }, + }, + }, + } + + ast, err := transpiler.NewAST(sConfig) + require.NoError(t, err) + + grouped, err := groupBy(ast) + require.NoError(t, err) + require.Equal(t, 2, len(grouped)) + + c1 := transpiler.MustNewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "xxx", + "username": "myusername", + "password": "mypassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/hello.log", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + map[string]interface{}{ + "type": "metrics/system", + "output": map[string]interface{}{ + "use_output": "special", + }, + }, + }, + }) + + c2, _ := transpiler.NewAST(map[string]interface{}{ + "output": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "yyy", + "username": "anotherusername", + "password": "anotherpassword", + }, + }, + "streams": []map[string]interface{}{ + map[string]interface{}{ + "type": "log", + "path": "/var/log/infosec.log", + "output": map[string]interface{}{ + "use_output": "infosec1", + "pipeline": "custompipeline", + "index_name": "myindex", + }, + }, + }, + }) + + defaultConfig, ok := grouped["special"] + require.True(t, ok) + require.Equal(t, c1.Hash(), defaultConfig.Hash()) + + infosec1Config, ok := grouped["infosec1"] + + require.True(t, ok) + require.Equal(t, c2.Hash(), infosec1Config.Hash()) + }) + + t.Run("copy any top level configuration options to each configuration", func(t *testing.T) { sConfig := map[string]interface{}{ "monitoring": map[string]interface{}{ "elasticsearch": map[string]interface{}{ - "hosts": "localhost", + "hosts": "127.0.0.1", }, }, + "keystore": map[string]interface{}{ + "path": "${path.data}/keystore", + }, "outputs": map[string]interface{}{ "special": map[string]interface{}{ "type": "elasticsearch", @@ -88,6 +194,14 @@ func TestGroupBy(t *testing.T) { }, }, }, + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "127.0.0.1", + }, + }, + "keystore": map[string]interface{}{ + "path": "${path.data}/keystore", + }, }) c2, _ := transpiler.NewAST(map[string]interface{}{ @@ -109,6 +223,14 @@ func TestGroupBy(t *testing.T) { }, }, }, + "monitoring": map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "hosts": "127.0.0.1", + }, + }, + "keystore": map[string]interface{}{ + "path": "${path.data}/keystore", + }, }) defaultConfig, ok := grouped["special"] @@ -121,8 +243,6 @@ func TestGroupBy(t *testing.T) { require.Equal(t, c2.Hash(), infosec1Config.Hash()) }) - t.Run("outputs with monitoring options", placeHolder) - t.Run("fail when the referenced named output doesn't exist", func(t *testing.T) { sConfig := map[string]interface{}{ "monitoring": map[string]interface{}{ @@ -173,11 +293,6 @@ func TestGroupBy(t *testing.T) { t.Run("only default output", func(t *testing.T) { sConfig := map[string]interface{}{ - "monitoring": map[string]interface{}{ - "elasticsearch": map[string]interface{}{ - "hosts": "localhost", - }, - }, "outputs": map[string]interface{}{ "default": map[string]interface{}{ "type": "elasticsearch", @@ -248,11 +363,6 @@ func TestGroupBy(t *testing.T) { t.Run("default and named output", func(t *testing.T) { sConfig := map[string]interface{}{ - "monitoring": map[string]interface{}{ - "elasticsearch": map[string]interface{}{ - "hosts": "localhost", - }, - }, "outputs": map[string]interface{}{ "default": map[string]interface{}{ "type": "elasticsearch", From 02d4dc877b21801e4766921b649ca1f9e9e9c7ac Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 12:14:45 -0500 Subject: [PATCH 05/13] move programs into his own methods --- x-pack/agent/pkg/agent/program/program.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index 5568f80065b4..ec3c8547b7f9 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -49,6 +49,10 @@ func (p *Program) Configuration() map[string]interface{} { // Programs take a Tree representation of the main configuration and apply all the different // programs rules and generate individual configuration from the rules. func Programs(singleConfig *transpiler.AST) ([]Program, error) { + return detectPrograms(singleConfig) +} + +func detectPrograms(singleConfig *transpiler.AST) ([]Program, error) { programs := make([]Program, 0) for _, spec := range Supported { // TODO: better error handling here. @@ -83,6 +87,7 @@ func Programs(singleConfig *transpiler.AST) ([]Program, error) { programs = append(programs, program) } return programs, nil + } // KnownProgramNames returns a list of runnable programs by the agent. From 4298f2ed6bb09e606a200cc12c524f5beb1647e0 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 12:49:18 -0500 Subject: [PATCH 06/13] WIP, fixing all the assertions --- x-pack/agent/pkg/agent/application/emitter.go | 6 +- x-pack/agent/pkg/agent/program/program.go | 19 +++- .../agent/pkg/agent/program/program_test.go | 98 +++++++++++++++++++ x-pack/agent/pkg/agent/program/spec_test.go | 89 ----------------- .../agent/program/testdata/audit_config.yml | 14 +-- .../agent/program/testdata/enabled_false.yml | 7 +- .../program/testdata/enabled_output_false.yml | 7 +- .../program/testdata/enabled_output_true.yml | 7 +- .../agent/program/testdata/enabled_true.yml | 7 +- .../agent/program/testdata/journal_config.yml | 7 +- .../agent/program/testdata/monitor_config.yml | 7 +- .../program/testdata/multiple_output_true.yml | 10 +- .../agent/program/testdata/single_config.yml | 7 +- 13 files changed, 157 insertions(+), 128 deletions(-) diff --git a/x-pack/agent/pkg/agent/application/emitter.go b/x-pack/agent/pkg/agent/application/emitter.go index 2e5997f50fef..4ec9abcbf151 100644 --- a/x-pack/agent/pkg/agent/application/emitter.go +++ b/x-pack/agent/pkg/agent/application/emitter.go @@ -41,10 +41,6 @@ func emitter(log *logger.Logger, router *router) emitterFunc { return err } - grouped := map[routingKey][]program.Program{ - defautlRK: programsToRun, - } - - return router.Dispatch(ast.HashStr(), grouped) + return router.Dispatch(ast.HashStr(), programsToRun) } } diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index ec3c8547b7f9..c796b065bfb7 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -48,14 +48,27 @@ func (p *Program) Configuration() map[string]interface{} { // Programs take a Tree representation of the main configuration and apply all the different // programs rules and generate individual configuration from the rules. -func Programs(singleConfig *transpiler.AST) ([]Program, error) { - return detectPrograms(singleConfig) +func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { + grouped, err := groupBy(singleConfig) + if err != nil { + return nil, errors.Wrap(err, "fail to extract program configuration") + } + + groupedPrograms := make(map[string][]Program) + for k, config := range grouped { + programs, err := detectPrograms(config) + if err != nil { + return nil, errors.Wrap(err, "fail to generate program configuration") + } + groupedPrograms[k] = programs + } + + return groupedPrograms, nil } func detectPrograms(singleConfig *transpiler.AST) ([]Program, error) { programs := make([]Program, 0) for _, spec := range Supported { - // TODO: better error handling here. specificAST := singleConfig.Clone() err := spec.Rules.Apply(specificAST) if err != nil { diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 72cb93f49cd1..48d82de0a04e 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -1,10 +1,17 @@ package program import ( + "io/ioutil" + "path/filepath" + "strings" "testing" + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + yaml "gopkg.in/yaml.v2" + "github.com/elastic/beats/x-pack/agent/pkg/agent/internal/yamltest" "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" ) @@ -454,3 +461,94 @@ func TestGroupBy(t *testing.T) { require.Equal(t, c2.Hash(), infosec1Config.Hash()) }) } + +func TestConfiguration(t *testing.T) { + testcases := map[string]struct { + programs []string + expected int + err bool + }{ + "single_config": { + programs: []string{"filebeat", "metricbeat"}, + expected: 2, + }, + "audit_config": { + programs: []string{"auditbeat"}, + expected: 1, + }, + "journal_config": { + programs: []string{"journalbeat"}, + expected: 1, + }, + "monitor_config": { + programs: []string{"heartbeat"}, + expected: 1, + }, + "enabled_true": { + programs: []string{"filebeat"}, + expected: 1, + }, + "enabled_false": { + expected: 0, + }, + "enabled_output_true": { + programs: []string{"filebeat"}, + expected: 1, + }, + "enabled_output_false": { + expected: 0, + }, + "multiple_output_true": { + err: true, + }, + } + + for name, test := range testcases { + t.Run(name, func(t *testing.T) { + singleConfig, err := ioutil.ReadFile(filepath.Join("testdata", name+".yml")) + require.NoError(t, err) + + var m map[string]interface{} + err = yaml.Unmarshal(singleConfig, &m) + require.NoError(t, err) + + ast, err := transpiler.NewAST(m) + require.NoError(t, err) + + programs, err := Programs(ast) + if test.err { + require.Error(t, err) + return + } + require.NoError(t, err) + + require.Equal(t, 1, len(programs)) + + defPrograms, ok := programs["default"] + require.True(t, ok) + require.Equal(t, test.expected, len(defPrograms)) + + for _, program := range defPrograms { + programConfig, err := ioutil.ReadFile(filepath.Join( + "testdata", + name+"-"+strings.ToLower(program.Spec.Name)+".yml", + )) + + require.NoError(t, err) + var m map[string]interface{} + err = yamltest.FromYAML(programConfig, &m) + require.NoError(t, err) + + compareMap := &transpiler.MapVisitor{} + program.Config.Accept(compareMap) + + if !assert.True(t, cmp.Equal(m, compareMap.Content)) { + diff := cmp.Diff(m, compareMap.Content) + if diff != "" { + t.Errorf("%s-%s mismatch (-want +got):\n%s", name, program.Spec.Name, diff) + } + } + } + }) + } +} diff --git a/x-pack/agent/pkg/agent/program/spec_test.go b/x-pack/agent/pkg/agent/program/spec_test.go index 5f964ad9231f..4c76f7214272 100644 --- a/x-pack/agent/pkg/agent/program/spec_test.go +++ b/x-pack/agent/pkg/agent/program/spec_test.go @@ -6,106 +6,17 @@ package program import ( "io/ioutil" - "path/filepath" "regexp" "strings" "testing" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" - "github.com/elastic/beats/x-pack/agent/pkg/agent/internal/yamltest" "github.com/elastic/beats/x-pack/agent/pkg/agent/transpiler" ) -func TestConfiguration(t *testing.T) { - testcases := map[string]struct { - programs []string - expected int - err bool - }{ - "single_config": { - programs: []string{"filebeat", "metricbeat"}, - expected: 2, - }, - "audit_config": { - programs: []string{"auditbeat"}, - expected: 1, - }, - "journal_config": { - programs: []string{"journalbeat"}, - expected: 1, - }, - "monitor_config": { - programs: []string{"heartbeat"}, - expected: 1, - }, - "enabled_true": { - programs: []string{"filebeat"}, - expected: 1, - }, - "enabled_false": { - expected: 0, - }, - "enabled_output_true": { - programs: []string{"filebeat"}, - expected: 1, - }, - "enabled_output_false": { - expected: 0, - }, - "multiple_output_true": { - err: true, - }, - } - - for name, test := range testcases { - t.Run(name, func(t *testing.T) { - singleConfig, err := ioutil.ReadFile(filepath.Join("testdata", name+".yml")) - require.NoError(t, err) - - var m map[string]interface{} - err = yaml.Unmarshal(singleConfig, &m) - require.NoError(t, err) - - ast, err := transpiler.NewAST(m) - require.NoError(t, err) - - programs, err := Programs(ast) - if test.err { - require.Error(t, err) - return - } - require.NoError(t, err) - require.Equal(t, test.expected, len(programs)) - - for _, program := range programs { - programConfig, err := ioutil.ReadFile(filepath.Join( - "testdata", - name+"-"+strings.ToLower(program.Spec.Name)+".yml", - )) - - require.NoError(t, err) - var m map[string]interface{} - err = yamltest.FromYAML(programConfig, &m) - require.NoError(t, err) - - compareMap := &transpiler.MapVisitor{} - program.Config.Accept(compareMap) - - if !assert.True(t, cmp.Equal(m, compareMap.Content)) { - diff := cmp.Diff(m, compareMap.Content) - if diff != "" { - t.Errorf("%s-%s mismatch (-want +got):\n%s", name, program.Spec.Name, diff) - } - } - } - }) - } -} - func TestSerialization(t *testing.T) { spec := Spec{ Name: "hello", diff --git a/x-pack/agent/pkg/agent/program/testdata/audit_config.yml b/x-pack/agent/pkg/agent/program/testdata/audit_config.yml index 03cafd93b461..b456eb492f13 100644 --- a/x-pack/agent/pkg/agent/program/testdata/audit_config.yml +++ b/x-pack/agent/pkg/agent/program/testdata/audit_config.yml @@ -1,4 +1,11 @@ -inputs: +outputs: + default: + type: "elasticsearch" + elasticsearch: + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme +streams: - type: log/file ignore_older: 123s - type: audit/auditd @@ -20,8 +27,3 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: - hosts: [127.0.0.1:9200, 127.0.0.1:9300] - username: elastic - password: changeme diff --git a/x-pack/agent/pkg/agent/program/testdata/enabled_false.yml b/x-pack/agent/pkg/agent/program/testdata/enabled_false.yml index 7ab37e596134..08ef6b70cd9d 100644 --- a/x-pack/agent/pkg/agent/program/testdata/enabled_false.yml +++ b/x-pack/agent/pkg/agent/program/testdata/enabled_false.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: event/file enabled: false paths: @@ -8,8 +8,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme diff --git a/x-pack/agent/pkg/agent/program/testdata/enabled_output_false.yml b/x-pack/agent/pkg/agent/program/testdata/enabled_output_false.yml index 143dc17eeb3a..94dc21d8bc8c 100644 --- a/x-pack/agent/pkg/agent/program/testdata/enabled_output_false.yml +++ b/x-pack/agent/pkg/agent/program/testdata/enabled_output_false.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: event/file paths: - /var/log/hello1.log @@ -7,8 +7,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch enabled: false hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic diff --git a/x-pack/agent/pkg/agent/program/testdata/enabled_output_true.yml b/x-pack/agent/pkg/agent/program/testdata/enabled_output_true.yml index d80e194c503a..9b3fb3c065aa 100644 --- a/x-pack/agent/pkg/agent/program/testdata/enabled_output_true.yml +++ b/x-pack/agent/pkg/agent/program/testdata/enabled_output_true.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: event/file paths: - /var/log/hello1.log @@ -7,8 +7,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch enabled: true hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic diff --git a/x-pack/agent/pkg/agent/program/testdata/enabled_true.yml b/x-pack/agent/pkg/agent/program/testdata/enabled_true.yml index c68bada60b94..ddbcb2929df5 100644 --- a/x-pack/agent/pkg/agent/program/testdata/enabled_true.yml +++ b/x-pack/agent/pkg/agent/program/testdata/enabled_true.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: event/file enabled: true paths: @@ -8,8 +8,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme diff --git a/x-pack/agent/pkg/agent/program/testdata/journal_config.yml b/x-pack/agent/pkg/agent/program/testdata/journal_config.yml index 7891b0ff7122..732ebab6fb2e 100644 --- a/x-pack/agent/pkg/agent/program/testdata/journal_config.yml +++ b/x-pack/agent/pkg/agent/program/testdata/journal_config.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: log/journal paths: [] backoff: 1s @@ -13,8 +13,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme diff --git a/x-pack/agent/pkg/agent/program/testdata/monitor_config.yml b/x-pack/agent/pkg/agent/program/testdata/monitor_config.yml index f663f6c48dde..97e5d9d8980a 100644 --- a/x-pack/agent/pkg/agent/program/testdata/monitor_config.yml +++ b/x-pack/agent/pkg/agent/program/testdata/monitor_config.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: monitor/icmp name: icmp schedule: '*/5 * * * * * *' @@ -15,8 +15,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme diff --git a/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml b/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml index 15519f04ad8e..4bdb68ef16b5 100644 --- a/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml +++ b/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: event/file paths: - /var/log/hello1.log @@ -7,12 +7,14 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch enabled: true hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme - logstash: + other: + type: logstash enabled: true hosts: [127.0.0.1:9200, 127.0.0.1:9300] diff --git a/x-pack/agent/pkg/agent/program/testdata/single_config.yml b/x-pack/agent/pkg/agent/program/testdata/single_config.yml index c89ce0b7a202..888a769b7f70 100644 --- a/x-pack/agent/pkg/agent/program/testdata/single_config.yml +++ b/x-pack/agent/pkg/agent/program/testdata/single_config.yml @@ -1,4 +1,4 @@ -inputs: +streams: - type: metric/docker setting: one - type: metric/apache @@ -11,8 +11,9 @@ management: host: "localhost" config: reload: 123 -output: - elasticsearch: +outputs: + default: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme From 8b0aa7203ae57bec7dd81d05a724c7f642f69491 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 13:24:57 -0500 Subject: [PATCH 07/13] update specs definition for streams. --- x-pack/agent/pkg/agent/program/program.go | 1 + x-pack/agent/pkg/agent/program/program_test.go | 6 +++--- x-pack/agent/pkg/agent/program/supported.go | 2 +- .../agent/pkg/agent/program/testdata/audit_config.yml | 3 +-- x-pack/agent/spec/auditbeat.yml | 10 +++++----- x-pack/agent/spec/filebeat.yml | 9 ++++++--- x-pack/agent/spec/heartbeat.yml | 9 ++++++--- x-pack/agent/spec/journalbeat.yml | 7 +++++-- x-pack/agent/spec/metricbeat.yml | 8 ++++---- 9 files changed, 32 insertions(+), 23 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index c796b065bfb7..9cf1234c3443 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -57,6 +57,7 @@ func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { groupedPrograms := make(map[string][]Program) for k, config := range grouped { programs, err := detectPrograms(config) + fmt.Println(programs) if err != nil { return nil, errors.Wrap(err, "fail to generate program configuration") } diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 48d82de0a04e..6d01c4dcfa40 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -498,9 +498,9 @@ func TestConfiguration(t *testing.T) { "enabled_output_false": { expected: 0, }, - "multiple_output_true": { - err: true, - }, + // "multiple_output_true": { //TODO NEED TO FIX IT + // err: true, + // }, } for name, test := range testcases { diff --git a/x-pack/agent/pkg/agent/program/supported.go b/x-pack/agent/pkg/agent/program/supported.go index b07271fbb48d..0cb7e31b1a0f 100644 --- a/x-pack/agent/pkg/agent/program/supported.go +++ b/x-pack/agent/pkg/agent/program/supported.go @@ -17,7 +17,7 @@ func init() { // spec/heartbeat.yml // spec/journalbeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJzMVlGTokYQfs/P2OdUDobTOlN1D8IeA+hixHUY5o2ZcQEdhCiikMp/T42goutebus2VXnYqnVgmu7+vq+//uthmy/Yp3DHk4IuwuK3KhUPvz/Q1CzIcxZNMVqF2EuIbypG6hbkMYuC7tnaUSlE9STKChuSnFqeYGIAAv+gEuy+cCCU0BxUxOdiYQ37RjKMbEPfELzq25bXY3A+so1hZFuOmCT6ook9LcfJtj13BYVoyeGgmiR6Tte6yq2n0XimFwE+x9sH2Mu6z9uYp9wi+Xs80xWWmjsGDjGH6I3cmj8GTSV8zCJuiT2ZZufz2zxHxufN6Oo5zzmM+rYxPRiJEgVgsF88Z00+kFQUKH0bih2zkML2zT0Ov/RtaKoECiVIUcyHspdqzB7be5YeH2N23zF6/1ZrRfxeTP35VV1k7ZT0+V5dXjlJ9IICT6VwPnp9X09ZOigmiR5z38s5dESAp7vQ7+059ur2HYXWWUT9wYr7B8H2HU4ss1d4sro9M5RrTjX5FjYcqNzSVT4817QhWNTyLpnmWgjRbpLodoBdh0NUsCoSC7W4yu+ci6oORrNh6jTvTwJfFUzT4wDMMwe03zHY1ja4oNCsORRLBlDMUjdzqr3Mc81SmePTemwM1yEw0xB8O/5P4WDNNC9mIFqPptnXh18bTb0kYnFHUl4qtsQ/pjcKgLqaJHoa+gfRQJ9Xx3Tr7AMoPv8xSkuqWfoFovM5AsTvKWNAcgrnfRseSrLv3JN0wkTQ9bRkmrcK/c9923IVAsWu+13b0AVP0Y4bAyXQhlL2Sza8H4f73n6S6Cqxhje5HEpSDVYUuBuC7b4NvTIAhWDRdRwJxVhzBIGixpq7pRrfUsD6Urby7CYv+axkmqjlvUmi1wvsyjPZe4mJQrDzwlNzy/1LjxlAW+K7CtXsV1IOAepNEl1ZYF20uLWj8Kl/ofWlj+OZXnPo5fSCX6d2JeLQ3V/uXdfcSNTLWSdWJ/8l1fTeKeeT7O/JlXT4eKnbPvGuJpKbmleydbcGyQn0EXK9cP880lz+MltFThJEM2jWz1I32N0HvitGBi859vYcT4/SI/4hZpqXB5orAuwsw0bCTV8qtpW5OaCISVrETrWPHHAoieYqAZayXo1OUo0X4eae/c0gitn62v7C7tkH2x/1B7sQeyVLvuxG++/y7Q2LuOYjW6PtFfcsrwr8Xk2hqRA00ELsZVjqBaDP51jG1bzpcLmdI+m8b5u9goJeLrEdV9Gn+ePh+N4fyVt26LVx3mOJscS0Y4k/ZiVX+PwcNzPim9Wt1WGs5LYRpPa3OGaKtEhUM2guySxaUw0pso6Ga2gbYFcJfbeWcQIQHTl7tpFjLnwT+GQTzI683VLAaw7NOjRYbkRfz1ayzHabdSjuMTQdqCztxRReOcqfVEMVTc3tHVf5bydbNTh/+31o5yXHzi7wDy1yynEycYiq1/k5Xce7qfV8fpoUF2YA1GNykqXzEQexoMvsOGFm2BMUP2UOLmh43c+PQb1B+jyVfhT1dFFsEnYH9GcfKSwVyxbcJfWnfRuqgltOHoCGCBfpXxqJNRJTCwmmviF9WIjFzXZ4kjvBXhX6T//7zZy2vXnfWDqNR0e0Pe6OwtPm+2oUNsRVS2KhLZnKEeWWzFrdbsw3tTebdtNP90x2lqKd3EpvLZt2sX619Xe3dlS/QzQ33/+pMXkVa5y2/cA/Z+NH625XuUZY37Hxv3/5JwAA//82Hfvb") + unpacked := packer.MustUnpack("eJzUV99zozYQfu+fkdd2Wn6c3aMz92DIIYMdXONEEnpDkgPYErgxNoFO//cOYGzsONdLk5lOHzyTCGn3291vv5X+vNluluyXcMeTnC7D/OdSipvfbqi0c3KfRXMM1yH2E4JsxZJeTm6zKOivpa5KAaxmUZY7gGzo2BdMGFqAnlWCvUeuCSW0jZIgLpbj0dBKRpFjmU8Er4fO2B8w8DBxrFHkjF0xS8xla3u+nybbw7onKIArDoxylpgVH7siQGplJUpEkV3Moiyq9zFgK+FtFjHdLwmyc1Yc1lO4Jfju4HeUO2O/DNCgosBWCDT0EPsZ1l1BNPiJ3Watz+ZnFgH2s1liKktsisP51qZ8GDr2IObI33DrczZNtpv+9ybm2yxyb79OpgtzRXVz0OEkqbun9y9xcvB56ABbJUAogYQxH9X5VOMjprEZcxCd77EGvXw0mEuOngWrujiUiEm4C5AqOv/ND5CSasp5/tuYFVplEUXGurZT5/jyfIe/z4Fp6ilMwpjiuwNWfz9LzJxqfsON2g6RYssBLI/10uCWIE+hult1a9OF2eYUuCLA82MMVIcKG0PlWEOQiyVuzgor5RlBn4bO1zhmilAIUquJBX9F2lmejlgwVjaOFch2/yCmCFYM2CuyiNLOj1sWkavBbYA9JUReRZBdBlqUTkc1Tl4S5G9YybaOxZ8CRJ6CRfP3lmq84sCuQottrOjLl5uf2t56TMTySmv5UmwJakMNNHU9S0wZomfRln9zLOV0YeYBPrZOR8t/LD0fuzFNvW2A/fPyd1R6Wf48wGZBsDPs09mxTMEl3HHLaPDNEnNLNdY7V1MKagQNlKnuKQSIXYMP+Bva91vTBxNB0/meA6+oWyvQR6/YgWt2m0Uc+cUFltr3ngBjFWqwloQ11bwngp0LO897UhptqVSjosDQKTDSWWK2a8U5rvrbVPcGTHvek6puT1GfSfvUxTqJ6RiecvyqNL0mcRfna2o3PjppanN3+h+u2ejYGr1cKVEX/2mvKwgQVc9WG0tRy4i3Z+N1xx/JpJFfwdu0Pulxcrqoc1W3yMPl2RNPX0pQYydEg4Jj/y2t3+P+cW3PsV/wnhSEGhww3d8z+TDhWizoKosWwK4W2BcU32Uuzmkvhl2H411tb41SJmu5u+sk4LvaPl6GT9dG6gLAmKXnIzXsr33wSKXI2IXY37Pk825SXNbAGda86+T7I7QG63zDQfzIJEwJjovv1J6yzg2W6p5KUePaT26N+e/tvh9fHbEHO28bs7EIsHsas2NXHGozORuP/X2v8PyY27TJXc1h5aSfZkVqTuvN99MV5Nz/u7je581Uil3NY6aqxmQxkm5iOgH2ZvX4ZroZB9pD5mqGysemyq2Gu4ICu+JArJgGYya9rOkBvdUStyxqLGmo2TLUvjZ9UOsV0/2YaVE6mWdHrq+y3VMaimtsl4bK5CCm4GzS/UF1WFJpb69Mu0MW3Ucu7S1HsHqpHOesbauWi+V9FvGxKMj8UO2L89NFOz3CDtPobeoYXsTyDYU8j+/jVPIib+9kz3k8H6OW//KSJJf5U8KuEOgeQYVJsToQZUXRfOgAVfCxuwm0thCnW/6Vca2+csu/QpjDzT4n2C9DdPd/eIUcsb5FLj/qdRBoRrG8lM3Lxjy8BlqcXjvivvGyoP16vy67DQZW9Ebk6uW17Oyqdcaj9zXOBcb/5nXx1w9/BwAA//9FElDj") for f, v := range unpacked { s, err := NewSpecFromBytes(v) diff --git a/x-pack/agent/pkg/agent/program/testdata/audit_config.yml b/x-pack/agent/pkg/agent/program/testdata/audit_config.yml index b456eb492f13..fbc963c97354 100644 --- a/x-pack/agent/pkg/agent/program/testdata/audit_config.yml +++ b/x-pack/agent/pkg/agent/program/testdata/audit_config.yml @@ -1,7 +1,6 @@ outputs: default: - type: "elasticsearch" - elasticsearch: + type: elasticsearch hosts: [127.0.0.1:9200, 127.0.0.1:9300] username: elastic password: changeme diff --git a/x-pack/agent/spec/auditbeat.yml b/x-pack/agent/spec/auditbeat.yml index c73e2909f0f7..c07f4a8d1794 100644 --- a/x-pack/agent/spec/auditbeat.yml +++ b/x-pack/agent/spec/auditbeat.yml @@ -4,25 +4,25 @@ rules: - filter_values_with_regexp: key: type re: ^audit/.+ - selector: inputs + selector: streams - map: - path: inputs + path: streams rules: - translate_with_regexp: path: type re: ^audit/(.+) with: $1 - copy: - from: inputs + from: streams to: auditbeat - map: - path: auditbeat.inputs + path: auditbeat.streams rules: - rename: from: type to: module - rename: - from: auditbeat.inputs + from: auditbeat.streams to: modules - filter: selectors: diff --git a/x-pack/agent/spec/filebeat.yml b/x-pack/agent/spec/filebeat.yml index 86a6032c085e..6e86f5808392 100644 --- a/x-pack/agent/spec/filebeat.yml +++ b/x-pack/agent/spec/filebeat.yml @@ -2,7 +2,7 @@ name: Filebeat cmd: filebeat rules: - map: - path: inputs + path: streams rules: - translate: path: type @@ -15,7 +15,7 @@ rules: log/redis_slowlog: redis log/syslog: syslog - filter_values: - selector: inputs + selector: streams key: type values: - log @@ -26,8 +26,11 @@ rules: - redis - syslog - copy: - from: inputs + from: streams to: filebeat +- rename: + from: filebeat.streams + to: inputs - filter: selectors: - filebeat diff --git a/x-pack/agent/spec/heartbeat.yml b/x-pack/agent/spec/heartbeat.yml index c939ebf9ab18..3da6bd485b29 100644 --- a/x-pack/agent/spec/heartbeat.yml +++ b/x-pack/agent/spec/heartbeat.yml @@ -4,17 +4,20 @@ rules: - filter_values_with_regexp: key: type re: ^monitor/.+ - selector: inputs + selector: streams - map: - path: inputs + path: streams rules: - translate_with_regexp: path: type re: ^monitor/(?P.+) with: $type - copy: - from: inputs + from: streams to: heartbeat +- rename: + from: heartbeat.streams + to: monitors - filter: selectors: - heartbeat diff --git a/x-pack/agent/spec/journalbeat.yml b/x-pack/agent/spec/journalbeat.yml index 40919e234015..6c5045b87bf8 100644 --- a/x-pack/agent/spec/journalbeat.yml +++ b/x-pack/agent/spec/journalbeat.yml @@ -2,13 +2,16 @@ name: Journalbeat cmd: journalbeat rules: - filter_values: - selector: inputs + selector: streams key: type values: - log/journal - copy: - from: inputs + from: streams to: journalbeat +- rename: + from: journalbeat.streams + to: inputs - filter: selectors: - journalbeat diff --git a/x-pack/agent/spec/metricbeat.yml b/x-pack/agent/spec/metricbeat.yml index a3a38a85d510..bc3a1eaba430 100644 --- a/x-pack/agent/spec/metricbeat.yml +++ b/x-pack/agent/spec/metricbeat.yml @@ -4,9 +4,9 @@ rules: - filter_values_with_regexp: key: type re: ^metric/.+ - selector: inputs + selector: streams - map: - path: inputs + path: streams rules: - translate_with_regexp: path: type @@ -16,10 +16,10 @@ rules: from: type to: module - copy: - from: inputs + from: streams to: metricbeat - rename: - from: metricbeat.inputs + from: metricbeat.streams to: modules - filter: selectors: From 920262864a147dde8bbae3c8e5b97db3f8fe3e39 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 14:33:59 -0500 Subject: [PATCH 08/13] make it work correctly with all the specs. --- x-pack/agent/pkg/agent/program/program.go | 1 - .../agent/pkg/agent/program/program_test.go | 3 --- x-pack/agent/pkg/agent/program/supported.go | 2 +- .../testdata/monitor_config-heartbeat.yml | 2 +- .../program/testdata/multiple_output_true.yml | 20 ------------------- x-pack/agent/pkg/agent/transpiler/ast.go | 9 ++++----- x-pack/agent/pkg/agent/transpiler/rules.go | 2 +- x-pack/agent/spec/heartbeat.yml | 4 ++-- 8 files changed, 9 insertions(+), 34 deletions(-) delete mode 100644 x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index 9cf1234c3443..c796b065bfb7 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -57,7 +57,6 @@ func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { groupedPrograms := make(map[string][]Program) for k, config := range grouped { programs, err := detectPrograms(config) - fmt.Println(programs) if err != nil { return nil, errors.Wrap(err, "fail to generate program configuration") } diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 6d01c4dcfa40..4f9b3c9464ce 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -498,9 +498,6 @@ func TestConfiguration(t *testing.T) { "enabled_output_false": { expected: 0, }, - // "multiple_output_true": { //TODO NEED TO FIX IT - // err: true, - // }, } for name, test := range testcases { diff --git a/x-pack/agent/pkg/agent/program/supported.go b/x-pack/agent/pkg/agent/program/supported.go index 0cb7e31b1a0f..1981afabd25b 100644 --- a/x-pack/agent/pkg/agent/program/supported.go +++ b/x-pack/agent/pkg/agent/program/supported.go @@ -17,7 +17,7 @@ func init() { // spec/heartbeat.yml // spec/journalbeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJzUV99zozYQfu+fkdd2Wn6c3aMz92DIIYMdXONEEnpDkgPYErgxNoFO//cOYGzsONdLk5lOHzyTCGn3291vv5X+vNluluyXcMeTnC7D/OdSipvfbqi0c3KfRXMM1yH2E4JsxZJeTm6zKOivpa5KAaxmUZY7gGzo2BdMGFqAnlWCvUeuCSW0jZIgLpbj0dBKRpFjmU8Er4fO2B8w8DBxrFHkjF0xS8xla3u+nybbw7onKIArDoxylpgVH7siQGplJUpEkV3Moiyq9zFgK+FtFjHdLwmyc1Yc1lO4Jfju4HeUO2O/DNCgosBWCDT0EPsZ1l1BNPiJ3Watz+ZnFgH2s1liKktsisP51qZ8GDr2IObI33DrczZNtpv+9ybm2yxyb79OpgtzRXVz0OEkqbun9y9xcvB56ABbJUAogYQxH9X5VOMjprEZcxCd77EGvXw0mEuOngWrujiUiEm4C5AqOv/ND5CSasp5/tuYFVplEUXGurZT5/jyfIe/z4Fp6ilMwpjiuwNWfz9LzJxqfsON2g6RYssBLI/10uCWIE+hult1a9OF2eYUuCLA82MMVIcKG0PlWEOQiyVuzgor5RlBn4bO1zhmilAIUquJBX9F2lmejlgwVjaOFch2/yCmCFYM2CuyiNLOj1sWkavBbYA9JUReRZBdBlqUTkc1Tl4S5G9YybaOxZ8CRJ6CRfP3lmq84sCuQottrOjLl5uf2t56TMTySmv5UmwJakMNNHU9S0wZomfRln9zLOV0YeYBPrZOR8t/LD0fuzFNvW2A/fPyd1R6Wf48wGZBsDPs09mxTMEl3HHLaPDNEnNLNdY7V1MKagQNlKnuKQSIXYMP+Bva91vTBxNB0/meA6+oWyvQR6/YgWt2m0Uc+cUFltr3ngBjFWqwloQ11bwngp0LO897UhptqVSjosDQKTDSWWK2a8U5rvrbVPcGTHvek6puT1GfSfvUxTqJ6RiecvyqNL0mcRfna2o3PjppanN3+h+u2ejYGr1cKVEX/2mvKwgQVc9WG0tRy4i3Z+N1xx/JpJFfwdu0Pulxcrqoc1W3yMPl2RNPX0pQYydEg4Jj/y2t3+P+cW3PsV/wnhSEGhww3d8z+TDhWizoKosWwK4W2BcU32Uuzmkvhl2H411tb41SJmu5u+sk4LvaPl6GT9dG6gLAmKXnIzXsr33wSKXI2IXY37Pk825SXNbAGda86+T7I7QG63zDQfzIJEwJjovv1J6yzg2W6p5KUePaT26N+e/tvh9fHbEHO28bs7EIsHsas2NXHGozORuP/X2v8PyY27TJXc1h5aSfZkVqTuvN99MV5Nz/u7je581Uil3NY6aqxmQxkm5iOgH2ZvX4ZroZB9pD5mqGysemyq2Gu4ICu+JArJgGYya9rOkBvdUStyxqLGmo2TLUvjZ9UOsV0/2YaVE6mWdHrq+y3VMaimtsl4bK5CCm4GzS/UF1WFJpb69Mu0MW3Ucu7S1HsHqpHOesbauWi+V9FvGxKMj8UO2L89NFOz3CDtPobeoYXsTyDYU8j+/jVPIib+9kz3k8H6OW//KSJJf5U8KuEOgeQYVJsToQZUXRfOgAVfCxuwm0thCnW/6Vca2+csu/QpjDzT4n2C9DdPd/eIUcsb5FLj/qdRBoRrG8lM3Lxjy8BlqcXjvivvGyoP16vy67DQZW9Ebk6uW17Oyqdcaj9zXOBcb/5nXx1w9/BwAA//9FElDj") + unpacked := packer.MustUnpack("eJzUV9Fzo7YTfv/9GXn9dVoMZ/fozD0YcgiIgxucSEJvSHIAW4BrsAl0+r93AGODY18vTWZu+uCZREi73+5++6305022WbJf/B2Pcrr085/LWNz8dkNjIyePafCA4drHbkSQIemxk5PbNPD6a4k9ogBW8yDNLUA21HQFE6rsoZcRwc4zl4XkG2pJEBdLczrRo2lg6dqW4PXEMt0xA093lj4NLNMW80hbtrYf9rMoO6w7ggK44kAt55FWcdMWHhpVeiQFFBnFPEiDeh8DhuTfpgFT3JIgI2fFYT2BGcH3B7/T3DLd0kPjigJDIlBVfOymWLEFkeEndpu2PpufVnjYTeeRJi2xJg7nW5vx08QyxiFH7obrn9NZlG3635uYb9PAvv16N1toK6po4w4nSew9fXyNk4PPEwsYIwKE5MUw5NM6n6PwiMnUQg6C4R593MtHg7nk6EWwqotDClgMdx4aic5/8wOkpLI0zH8bs0SrNKBIXdd26hyfn+/w9zkwSxyJxTCk+P6A1d3PIy2nsttwo7ZDYpFxAMtjvWSYEeRIVLGrbm220NqcAlt4+OEYA1WgxEwoHWsIcrHEzVmhJzwl6NPE+hqGTBISQaPqToe/InmQpyMWjKWNpXtxu38cUgQrBowVWQRJ58cui8CWYeZhR/KRUxFklJ4cJLNpjZOXBLkbVrLM0vnWQ2TrLZq/MyrzigOj8nW20YMvX25+anvrORLLC63lxiIjqA3Vk0freaTFPnoRbfk3x1LOFlru4WPrdLT8x9Jz0w5p4mQedofl76j0uvy5h7WCYGvSp7Ola4LHcMd1tcE3j7SMyqx3rqYUlAkaSzPFkQgQuwYfcDe077emDyaCJg97Dpyibi1PmV6xA9fsNg04coszLLXvPQHqypdhLQlrKjtbgq0zOy97UqptqUZqRYGqUKAm80hr14ohrvrbTHHGTH7Zk6puT1GfSfrUxQoJqQlPOb4qTdck7ux8Te3GRydNbe5O/8M1mx5bo5crKejiP+21BQGi6tlqYylqGXH2zFx3/IlZrOYX8DatT3qcnC3qXNUt8nR+9sTT1xLU2PHRuODYfUvr97h/XNtz7Ba8JwW+DMdMcfcsfrrjcijoKg0WwKgW2BUU36c2zmkvhl2H411tr08TFtdyd99JwHe1fbj0t5dG6gLAkCXDker31z54pFKk7nzs7ln0eXdXnNfAmtS86+T7I7QGK3zDQfjMYpgQHBbfqT1lnRscj/Y0FjWu/d2t+vB7u+//V0fswc7bxmwoPGyfxqxpi0Nt7gbjsb/vCs+PuU2a3NUclk76qVWk5rTSfD9dQYb+38X1Pm9mx9zZP4Tvq3S3TXxxifGxOmLxOKRgMO3+oAosaWxkFybeIZP2M4+NjCNYvVaPIXPbyuVi+ZgG3BQFeThU/Oz8bNFOEL/DNH2bQvpnsXxDJYfxfZxSnuXtnQwaxvMxivkvL0rxMt9G7AKBHhGUWCxWB6KsKHqYWGAkuGlvPLktxOmmf2Fkj67c9C8Q5nC7zwl2Sx/d/xdeIkesb5HMj3oheLJaLM+l87wxDy+CFqfTjrlvvC5ov97XpbfBwIremFy9vpoNrlsDHr2vcc4w/pgXxl//+zsAAP//k9VT0g==") for f, v := range unpacked { s, err := NewSpecFromBytes(v) diff --git a/x-pack/agent/pkg/agent/program/testdata/monitor_config-heartbeat.yml b/x-pack/agent/pkg/agent/program/testdata/monitor_config-heartbeat.yml index d9215783a71b..59feb7568854 100644 --- a/x-pack/agent/pkg/agent/program/testdata/monitor_config-heartbeat.yml +++ b/x-pack/agent/pkg/agent/program/testdata/monitor_config-heartbeat.yml @@ -1,5 +1,5 @@ heartbeat: - inputs: + monitors: - type: icmp name: icmp schedule: "*/5 * * * * * *" diff --git a/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml b/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml deleted file mode 100644 index 4bdb68ef16b5..000000000000 --- a/x-pack/agent/pkg/agent/program/testdata/multiple_output_true.yml +++ /dev/null @@ -1,20 +0,0 @@ -streams: - - type: event/file - paths: - - /var/log/hello1.log - - /var/log/hello2.log -management: - host: "localhost" -config: - reload: 123 -outputs: - default: - type: elasticsearch - enabled: true - hosts: [127.0.0.1:9200, 127.0.0.1:9300] - username: elastic - password: changeme - other: - type: logstash - enabled: true - hosts: [127.0.0.1:9200, 127.0.0.1:9300] diff --git a/x-pack/agent/pkg/agent/transpiler/ast.go b/x-pack/agent/pkg/agent/transpiler/ast.go index 929776eb3474..d97b0fa009f4 100644 --- a/x-pack/agent/pkg/agent/transpiler/ast.go +++ b/x-pack/agent/pkg/agent/transpiler/ast.go @@ -66,11 +66,10 @@ type Dict struct { // Find takes a string which is a key and try to find the elements in the associated K/V. func (d *Dict) Find(key string) (Node, bool) { - i := sort.Search(len(d.value), func(i int) bool { - return d.value[i].(*Key).name >= key - }) - if i < len(d.value) && d.value[i].(*Key).name == key { - return d.value[i], true + for _, i := range d.value { + if i.(*Key).name == key { + return i, true + } } return nil, false } diff --git a/x-pack/agent/pkg/agent/transpiler/rules.go b/x-pack/agent/pkg/agent/transpiler/rules.go index 198ca5f914d7..9f8e497a38de 100644 --- a/x-pack/agent/pkg/agent/transpiler/rules.go +++ b/x-pack/agent/pkg/agent/transpiler/rules.go @@ -424,7 +424,7 @@ func (r *FilterRule) Apply(ast *AST) error { mergedAST := &AST{root: &Dict{}} var err error for _, selector := range r.Selectors { - newAST, ok := Select(ast, selector) + newAST, ok := Select(ast.Clone(), selector) if !ok { continue } diff --git a/x-pack/agent/spec/heartbeat.yml b/x-pack/agent/spec/heartbeat.yml index 3da6bd485b29..14aaa1d3da47 100644 --- a/x-pack/agent/spec/heartbeat.yml +++ b/x-pack/agent/spec/heartbeat.yml @@ -23,5 +23,5 @@ rules: - heartbeat - output - keystore -when: HasItems(%{[heartbeat.inputs]}) && HasNamespace('output', 'elasticsearch', 'redis', - 'kafka', 'logstash') +when: HasItems(%{[heartbeat.monitors]}) && HasNamespace('output', 'elasticsearch', + 'redis', 'kafka', 'logstash') From 20624a612804f184aeb38cd70346177ada6a72cc Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 14:42:51 -0500 Subject: [PATCH 09/13] revert --- x-pack/agent/pkg/agent/transpiler/ast.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/x-pack/agent/pkg/agent/transpiler/ast.go b/x-pack/agent/pkg/agent/transpiler/ast.go index d97b0fa009f4..d0613e2b775c 100644 --- a/x-pack/agent/pkg/agent/transpiler/ast.go +++ b/x-pack/agent/pkg/agent/transpiler/ast.go @@ -87,11 +87,6 @@ func (d *Dict) String() string { return sb.String() } -// Len returns the number of items in the dictionnary. -func (d *Dict) Len() int { - return len(d.value) -} - // Value returns the value of dict which is a slice of node. func (d *Dict) Value() interface{} { return d.value @@ -159,11 +154,6 @@ func (k *Key) Hash() []byte { return h.Sum(nil) } -// Name return the name of the key. -func (k *Key) Name() string { - return k.name -} - // List represents a slice in our Tree. type List struct { value []Node @@ -182,11 +172,6 @@ func (l *List) String() string { return sb.String() } -// Len returns the number of items in the list. -func (l *List) Len() int { - return len(l.value) -} - // Hash compute a sha256 hash of the current node and recursively call any children. func (l *List) Hash() []byte { h := sha256.New() From de9eb750fd3356f07fdde6cf1716247f60a802d0 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Thu, 28 Nov 2019 14:48:55 -0500 Subject: [PATCH 10/13] supported files --- x-pack/agent/pkg/agent/program/supported.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/agent/pkg/agent/program/supported.go b/x-pack/agent/pkg/agent/program/supported.go index 1981afabd25b..e555b05884f9 100644 --- a/x-pack/agent/pkg/agent/program/supported.go +++ b/x-pack/agent/pkg/agent/program/supported.go @@ -17,7 +17,7 @@ func init() { // spec/heartbeat.yml // spec/journalbeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJzUV9Fzo7YTfv/9GXn9dVoMZ/fozD0YcgiIgxucSEJvSHIAW4BrsAl0+r93AGODY18vTWZu+uCZREi73+5++6305022WbJf/B2Pcrr085/LWNz8dkNjIyePafCA4drHbkSQIemxk5PbNPD6a4k9ogBW8yDNLUA21HQFE6rsoZcRwc4zl4XkG2pJEBdLczrRo2lg6dqW4PXEMt0xA093lj4NLNMW80hbtrYf9rMoO6w7ggK44kAt55FWcdMWHhpVeiQFFBnFPEiDeh8DhuTfpgFT3JIgI2fFYT2BGcH3B7/T3DLd0kPjigJDIlBVfOymWLEFkeEndpu2PpufVnjYTeeRJi2xJg7nW5vx08QyxiFH7obrn9NZlG3635uYb9PAvv16N1toK6po4w4nSew9fXyNk4PPEwsYIwKE5MUw5NM6n6PwiMnUQg6C4R593MtHg7nk6EWwqotDClgMdx4aic5/8wOkpLI0zH8bs0SrNKBIXdd26hyfn+/w9zkwSxyJxTCk+P6A1d3PIy2nsttwo7ZDYpFxAMtjvWSYEeRIVLGrbm220NqcAlt4+OEYA1WgxEwoHWsIcrHEzVmhJzwl6NPE+hqGTBISQaPqToe/InmQpyMWjKWNpXtxu38cUgQrBowVWQRJ58cui8CWYeZhR/KRUxFklJ4cJLNpjZOXBLkbVrLM0vnWQ2TrLZq/MyrzigOj8nW20YMvX25+anvrORLLC63lxiIjqA3Vk0freaTFPnoRbfk3x1LOFlru4WPrdLT8x9Jz0w5p4mQedofl76j0uvy5h7WCYGvSp7Ola4LHcMd1tcE3j7SMyqx3rqYUlAkaSzPFkQgQuwYfcDe077emDyaCJg97Dpyibi1PmV6xA9fsNg04coszLLXvPQHqypdhLQlrKjtbgq0zOy97UqptqUZqRYGqUKAm80hr14ohrvrbTHHGTH7Zk6puT1GfSfrUxQoJqQlPOb4qTdck7ux8Te3GRydNbe5O/8M1mx5bo5crKejiP+21BQGi6tlqYylqGXH2zFx3/IlZrOYX8DatT3qcnC3qXNUt8nR+9sTT1xLU2PHRuODYfUvr97h/XNtz7Ba8JwW+DMdMcfcsfrrjcijoKg0WwKgW2BUU36c2zmkvhl2H411tr08TFtdyd99JwHe1fbj0t5dG6gLAkCXDker31z54pFKk7nzs7ln0eXdXnNfAmtS86+T7I7QGK3zDQfjMYpgQHBbfqT1lnRscj/Y0FjWu/d2t+vB7u+//V0fswc7bxmwoPGyfxqxpi0Nt7gbjsb/vCs+PuU2a3NUclk76qVWk5rTSfD9dQYb+38X1Pm9mx9zZP4Tvq3S3TXxxifGxOmLxOKRgMO3+oAosaWxkFybeIZP2M4+NjCNYvVaPIXPbyuVi+ZgG3BQFeThU/Oz8bNFOEL/DNH2bQvpnsXxDJYfxfZxSnuXtnQwaxvMxivkvL0rxMt9G7AKBHhGUWCxWB6KsKHqYWGAkuGlvPLktxOmmf2Fkj67c9C8Q5nC7zwl2Sx/d/xdeIkesb5HMj3oheLJaLM+l87wxDy+CFqfTjrlvvC5ov97XpbfBwIremFy9vpoNrlsDHr2vcc4w/pgXxl//+zsAAP//k9VT0g==") + unpacked := packer.MustUnpack("eJzUV9Fzo7YTfv/9GXn9dVoMZ/fozD0YcgiIgxucSEJvSHIAW4BrsAl0+r93AGODY18vTWZu+uCZREi73+5++6305022WbJf/B2Pcrr085/LWNz8dkNjIyePafCA4drHbkSQIemxk5PbNPD6a4k9ogBW8yDNLUA21HQFE6rsoZcRwc4zl4XkG2pJEBdLczrRo2lg6dqW4PXEMt0xA093lj4NLNMW80hbtrYf9rMoO6w7ggK44kAt55FWcdMWHhpVeiQFFBnFPEiDeh8DhuTfpgFT3JIgI2fFYT2BGcH3B7/T3DLd0kPjigJDIlBVfOymWLEFkeEndpu2PpufVnjYTeeRJi2xJg7nW5vx08QyxiFH7obrn9NZlG3635uYb9PAvv16N1toK6po4w4nSew9fXyNk4PPEwsYIwKE5MUw5NM6n6PwiMnUQg6C4R593MtHg7nk6EWwqotDClgMdx4aic5/8wOkpLI0zH8bs0SrNKBIXdd26hyfn+/w9zkwSxyJxTCk+P6A1d3PIy2nsttwo7ZDYpFxAMtjvWSYEeRIVLGrbm220NqcAlt4+OEYA1WgxEwoHWsIcrHEzVmhJzwl6NPE+hqGTBISQaPqToe/InmQpyMWjKWNpXtxu38cUgQrBowVWQRJ58cui8CWYeZhR/KRUxFklJ4cJLNpjZOXBLkbVrLM0vnWQ2TrLZq/MyrzigOj8nW20YMvX25+anvrORLLC63lxiIjqA3Vk0freaTFPnoRbfk3x1LOFlru4WPrdLT8x9Jz0w5p4mQedofl76j0uvy5h7WCYGvSp7Ola4LHcMd1tcE3j7SMyqx3rqYUlAkaSzPFkQgQuwYfcDe077emDyaCJg97Dpyibi1PmV6xA9fsNg04coszLLXvPQHqypdhLQlrKjtbgq0zOy97UqptqUZqRYGqUKAm80hr14ohrvrbTHHGTH7Zk6puT1GfSfrUxQoJqQlPOb4qTdck7ux8Te3GRydNbe5O/8M1mx5bo5crKejiP+21BQGi6tlqYylqGXH2zFx3/IlZrOYX8DatT3qcnC3qXNUt8nR+9sTT1xLU2PHRuODYfUvr97h/XNtz7Ba8JwW+DMdMcfcsfrrjcijoKg0WwKgW2BUU36c2zmkvhl2H411tr08TFtdyd99JwHe1fbj0t5dG6gLAkCXDker31z54pFKk7nzs7ln0eXdXnNfAmtS86+T7I7QGK3zDQfjMYpgQHBbfqT1lnRscj/Y0FjWu/d2t+vB7u+//V0fswc7bxmwoPGyfxqxpi0Nt7gbjsb/vCs+PuU2a3NUclk76qVWk5rTSfD9dQYb+38X1Pm9mx9zZP2TMrdLdNvHFJcbH6ojF45CCwbT7gyqwpLGRXZh4h0zazzw2Mo5g9Vo9hsxtK5eL5WMacFMU5OFQ8bPzs0U7QfwO0/RtCumfxfINlRzG93FKeZa3dzJoGM/HKOa/ZFC8zLcRu0CgRwQlFovVgSgrih4mFhgJbtobT24LcbrpXxjZoys3/QuEOdzuc4Ld0kf3/4WXyBHrWyTzo14InqwWy3PpPG/Mw4ugxem0Y+4brwvar/d16W0wsKI3Jlevr2aD69aAR+9rnDOMP+aF8df//g4AAP//Oi1UHQ==") for f, v := range unpacked { s, err := NewSpecFromBytes(v) From 07cced6435268d6cd438f974d457d12f01ef655f Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 2 Dec 2019 09:54:07 -0500 Subject: [PATCH 11/13] reviews comments --- x-pack/agent/pkg/agent/program/program.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index c796b065bfb7..8d687a0a375e 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -49,7 +49,7 @@ func (p *Program) Configuration() map[string]interface{} { // Programs take a Tree representation of the main configuration and apply all the different // programs rules and generate individual configuration from the rules. func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { - grouped, err := groupBy(singleConfig) + grouped, err := groupByOutput(singleConfig) if err != nil { return nil, errors.Wrap(err, "fail to extract program configuration") } @@ -112,7 +112,7 @@ func KnownProgramNames() []string { return names } -func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { +func groupByOutput(single *transpiler.AST) (map[string]*transpiler.AST, error) { const ( outputsKey = "outputs" outputKey = "output" @@ -139,7 +139,13 @@ func groupBy(single *transpiler.AST) (map[string]*transpiler.AST, error) { return nil, errors.New("fail to received a list of configured outputs") } - out := m.(map[string]interface{}) + out, ok := m.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf( + "invalid outputs configuration received, expecting a map not a %T", + m, + ) + } for k, v := range out { outputsOptions, ok := v.(map[string]interface{}) From e8a712cabe320f23b73dab478e3fbc4ca9debae3 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 2 Dec 2019 14:43:15 -0500 Subject: [PATCH 12/13] should be plurial --- x-pack/agent/pkg/agent/program/program.go | 4 ++-- x-pack/agent/pkg/agent/program/program_test.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/x-pack/agent/pkg/agent/program/program.go b/x-pack/agent/pkg/agent/program/program.go index 8d687a0a375e..9266786967fc 100644 --- a/x-pack/agent/pkg/agent/program/program.go +++ b/x-pack/agent/pkg/agent/program/program.go @@ -49,7 +49,7 @@ func (p *Program) Configuration() map[string]interface{} { // Programs take a Tree representation of the main configuration and apply all the different // programs rules and generate individual configuration from the rules. func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { - grouped, err := groupByOutput(singleConfig) + grouped, err := groupByOutputs(singleConfig) if err != nil { return nil, errors.Wrap(err, "fail to extract program configuration") } @@ -112,7 +112,7 @@ func KnownProgramNames() []string { return names } -func groupByOutput(single *transpiler.AST) (map[string]*transpiler.AST, error) { +func groupByOutputs(single *transpiler.AST) (map[string]*transpiler.AST, error) { const ( outputsKey = "outputs" outputKey = "output" diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 4f9b3c9464ce..4f1868ade7d8 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -61,7 +61,7 @@ func TestGroupBy(t *testing.T) { ast, err := transpiler.NewAST(sConfig) require.NoError(t, err) - grouped, err := groupBy(ast) + grouped, err := groupByOutputs(ast) require.NoError(t, err) require.Equal(t, 2, len(grouped)) @@ -174,7 +174,7 @@ func TestGroupBy(t *testing.T) { ast, err := transpiler.NewAST(sConfig) require.NoError(t, err) - grouped, err := groupBy(ast) + grouped, err := groupByOutputs(ast) require.NoError(t, err) require.Equal(t, 2, len(grouped)) @@ -294,7 +294,7 @@ func TestGroupBy(t *testing.T) { ast, err := transpiler.NewAST(sConfig) require.NoError(t, err) - _, err = groupBy(ast) + _, err = groupByOutputs(ast) require.Error(t, err) }) @@ -332,7 +332,7 @@ func TestGroupBy(t *testing.T) { ast, err := transpiler.NewAST(sConfig) require.NoError(t, err) - grouped, err := groupBy(ast) + grouped, err := groupByOutputs(ast) require.NoError(t, err) require.Equal(t, 1, len(grouped)) @@ -407,7 +407,7 @@ func TestGroupBy(t *testing.T) { ast, err := transpiler.NewAST(sConfig) require.NoError(t, err) - grouped, err := groupBy(ast) + grouped, err := groupByOutputs(ast) require.NoError(t, err) require.Equal(t, 2, len(grouped)) From 07a1dc2e7e86bc99653b09205cd2e61ec306fbb6 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Tue, 3 Dec 2019 11:12:47 -0500 Subject: [PATCH 13/13] missing license --- x-pack/agent/pkg/agent/program/program_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/agent/pkg/agent/program/program_test.go b/x-pack/agent/pkg/agent/program/program_test.go index 4f1868ade7d8..2929fa6354cd 100644 --- a/x-pack/agent/pkg/agent/program/program_test.go +++ b/x-pack/agent/pkg/agent/program/program_test.go @@ -1,3 +1,7 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + package program import (