From c01bbf2951ccddecab6f868d6515ed7fd283249b Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Mon, 16 Jan 2023 13:16:28 +0100 Subject: [PATCH 1/5] Add support for optional metricsets in xpack mode --- metricbeat/helper/elastic/elastic.go | 29 +++++++++++++++++-- metricbeat/helper/elastic/elastic_test.go | 27 ++++++++++++++--- metricbeat/module/beat/beat.go | 2 +- .../module/elasticsearch/elasticsearch.go | 2 +- metricbeat/module/kibana/kibana.go | 2 +- metricbeat/module/logstash/logstash.go | 2 +- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index fa466a5fee1c..c23ca3690205 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -138,7 +138,7 @@ func FixTimestampField(m mapstr.M, field string) error { } // NewModule returns a new Elastic stack module with the appropriate metricsets configured. -func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) { +func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpackMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) { moduleName := base.Name() config := struct { @@ -158,8 +158,31 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *log return nil, errors.Wrapf(err, "could not unpack configuration for module %v", moduleName) } - // These metricsets are exactly the ones required if xpack.enabled == true - raw["metricsets"] = xpackEnabledMetricsets + // Ensure all required metricsets are enabled when xpack.enabled == true + cfgdMetricsets, err := raw.GetValue("metricsets") + if err != nil || cfgdMetricsets == nil || len(cfgdMetricsets.([]interface{})) == 0 { + raw["metricsets"] = xpackEnabledMetricsets + } else { + // Allow some optional metricsets to be enabled when xpack.enabled == true + cfgdMetricsetsSlice := cfgdMetricsets.([]interface{}) + metricsets := xpackEnabledMetricsets + + for _, cfgdMs := range cfgdMetricsetsSlice { + found := false + for _, optMs := range optionalXpackMetricsets { + if cfgdMs == optMs { + found = true + break + } + } + + if found { + metricsets = append(metricsets, cfgdMs.(string)) + } + } + + raw["metricsets"] = metricsets + } newConfig, err := conf.NewConfigFrom(raw) if err != nil { diff --git a/metricbeat/helper/elastic/elastic_test.go b/metricbeat/helper/elastic/elastic_test.go index a7cc38a5680e..be5529928029 100644 --- a/metricbeat/helper/elastic/elastic_test.go +++ b/metricbeat/helper/elastic/elastic_test.go @@ -157,11 +157,14 @@ func TestConfigureModule(t *testing.T) { require.NoError(t, err) err = mockRegistry.AddMetricSet(moduleName, "baz", mockMetricSetFactory) require.NoError(t, err) + err = mockRegistry.AddMetricSet(moduleName, "thud", mockMetricSetFactory) + require.NoError(t, err) tests := map[string]struct { - initConfig metricSetConfig - xpackEnabledMetricsets []string - newConfig metricSetConfig + initConfig metricSetConfig + xpackEnabledMetricsets []string + optionalXpackMetricsets []string + newConfig metricSetConfig }{ "no_xpack_enabled": { metricSetConfig{ @@ -169,6 +172,7 @@ func TestConfigureModule(t *testing.T) { MetricSets: []string{"foo", "bar"}, }, []string{"baz", "qux", "foo"}, + []string{}, metricSetConfig{ Module: moduleName, MetricSets: []string{"foo", "bar"}, @@ -181,12 +185,27 @@ func TestConfigureModule(t *testing.T) { MetricSets: []string{"foo", "bar"}, }, []string{"baz", "qux", "foo"}, + []string{}, metricSetConfig{ Module: moduleName, XPackEnabled: true, MetricSets: []string{"baz", "qux", "foo"}, }, }, + "xpack_enabled_with_optional": { + metricSetConfig{ + Module: moduleName, + XPackEnabled: true, + MetricSets: []string{"foo", "bar", "thud"}, + }, + []string{"baz", "qux", "foo"}, + []string{"bar"}, + metricSetConfig{ + Module: moduleName, + XPackEnabled: true, + MetricSets: []string{"baz", "qux", "foo", "bar"}, // include optional, exclude others + }, + }, } for name, test := range tests { @@ -200,7 +219,7 @@ func TestConfigureModule(t *testing.T) { require.Fail(t, "expecting module to be base module") } - newM, err := NewModule(bm, test.xpackEnabledMetricsets, logp.L()) + newM, err := NewModule(bm, test.xpackEnabledMetricsets, test.optionalXpackMetricsets, logp.L()) require.NoError(t, err) var newConfig metricSetConfig diff --git a/metricbeat/module/beat/beat.go b/metricbeat/module/beat/beat.go index c58e33974454..138e2d99750b 100644 --- a/metricbeat/module/beat/beat.go +++ b/metricbeat/module/beat/beat.go @@ -41,7 +41,7 @@ var metricSets = []string{"state", "stats"} // NewModule creates a new module func NewModule(base mb.BaseModule) (mb.Module, error) { - return elastic.NewModule(&base, metricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, metricSets, []string{}, logp.NewLogger(ModuleName)) } // ModuleName is the name of this module. diff --git a/metricbeat/module/elasticsearch/elasticsearch.go b/metricbeat/module/elasticsearch/elasticsearch.go index cd7dc0ec47ed..56e2d5bedf87 100644 --- a/metricbeat/module/elasticsearch/elasticsearch.go +++ b/metricbeat/module/elasticsearch/elasticsearch.go @@ -56,7 +56,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { "node_stats", "shard", } - return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName)) } var ( diff --git a/metricbeat/module/kibana/kibana.go b/metricbeat/module/kibana/kibana.go index 781a0fccfda9..9a3fa60b12e8 100644 --- a/metricbeat/module/kibana/kibana.go +++ b/metricbeat/module/kibana/kibana.go @@ -92,7 +92,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) { xpackEnabledMetricSets := []string{ "stats", "cluster_rules", "node_rules", "cluster_actions", "node_actions", } - return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName)) } // GetVersion returns the version of the Kibana instance diff --git a/metricbeat/module/logstash/logstash.go b/metricbeat/module/logstash/logstash.go index 6008b8d947ad..8623bcd9a803 100644 --- a/metricbeat/module/logstash/logstash.go +++ b/metricbeat/module/logstash/logstash.go @@ -40,7 +40,7 @@ func init() { // NewModule creates a new module func NewModule(base mb.BaseModule) (mb.Module, error) { - return elastic.NewModule(&base, []string{"node", "node_stats"}, logp.NewLogger(ModuleName)) + return elastic.NewModule(&base, []string{"node", "node_stats"}, []string{}, logp.NewLogger(ModuleName)) } // ModuleName is the name of this module. From afd63ff88a8f2c41cc135c69ec26dcbea75d0d46 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Fri, 20 Jan 2023 12:37:04 +0100 Subject: [PATCH 2/5] Simplify metricset loop --- metricbeat/helper/elastic/elastic.go | 33 +++++++++++++--------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index c23ca3690205..39a4d342e601 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/pkg/errors" + "golang.org/x/exp/slices" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -158,32 +159,28 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpa return nil, errors.Wrapf(err, "could not unpack configuration for module %v", moduleName) } - // Ensure all required metricsets are enabled when xpack.enabled == true + // Ensure all required metricsets are enabled when xpack.enabled == true, and add any additional which are optional cfgdMetricsets, err := raw.GetValue("metricsets") - if err != nil || cfgdMetricsets == nil || len(cfgdMetricsets.([]interface{})) == 0 { - raw["metricsets"] = xpackEnabledMetricsets - } else { - // Allow some optional metricsets to be enabled when xpack.enabled == true + metricsets := xpackEnabledMetricsets + if err == nil && cfgdMetricsets != nil { + // Type cast the metricsets to a slice of strings cfgdMetricsetsSlice := cfgdMetricsets.([]interface{}) - metricsets := xpackEnabledMetricsets - - for _, cfgdMs := range cfgdMetricsetsSlice { - found := false - for _, optMs := range optionalXpackMetricsets { - if cfgdMs == optMs { - found = true - break - } - } + cfgdMetricsetsStrings := make([]string, len(cfgdMetricsetsSlice)) + for i := range cfgdMetricsetsSlice { + cfgdMetricsetsStrings[i] = cfgdMetricsetsSlice[i].(string) + } - if found { - metricsets = append(metricsets, cfgdMs.(string)) + // Add any optional metricsets which are not already configured + for _, cfgdMs := range cfgdMetricsetsStrings { + if slices.Contains(optionalXpackMetricsets, cfgdMs) { + metricsets = append(metricsets, cfgdMs) } } - raw["metricsets"] = metricsets } + raw["metricsets"] = metricsets + newConfig, err := conf.NewConfigFrom(raw) if err != nil { return nil, errors.Wrapf(err, "could not create new configuration for module %v", moduleName) From 60b85e37ccf00aa3f38801ab468a12a62531d5e0 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Tue, 24 Jan 2023 10:19:38 +0100 Subject: [PATCH 3/5] go mod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index de23281476a2..80d7028b648b 100644 --- a/go.mod +++ b/go.mod @@ -209,6 +209,7 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.0.0 go.elastic.co/apm/v2 v2.0.0 go.mongodb.org/mongo-driver v1.5.1 + golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -330,7 +331,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.opencensus.io v0.23.0 // indirect - golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect golang.org/x/term v0.2.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect From b086da76c89cd3cc75f0bf35b2ba7fb07558caee Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Tue, 24 Jan 2023 10:23:42 +0100 Subject: [PATCH 4/5] Remove dependency on exp/slices --- go.mod | 2 +- metricbeat/helper/elastic/elastic.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 80d7028b648b..de23281476a2 100644 --- a/go.mod +++ b/go.mod @@ -209,7 +209,6 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.0.0 go.elastic.co/apm/v2 v2.0.0 go.mongodb.org/mongo-driver v1.5.1 - golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -331,6 +330,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.2 // indirect go.elastic.co/fastjson v1.1.0 // indirect go.opencensus.io v0.23.0 // indirect + golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect golang.org/x/term v0.2.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index 39a4d342e601..3000270f4f5e 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -22,7 +22,6 @@ import ( "strings" "github.com/pkg/errors" - "golang.org/x/exp/slices" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -172,7 +171,15 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpa // Add any optional metricsets which are not already configured for _, cfgdMs := range cfgdMetricsetsStrings { - if slices.Contains(optionalXpackMetricsets, cfgdMs) { + found := false + for _, ms := range optionalXpackMetricsets { + if ms == cfgdMs { + found = true + break + } + } + + if found { metricsets = append(metricsets, cfgdMs) } } From 729782e9ad5d45108f33e172c6ffdae9e66e851b Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Tue, 24 Jan 2023 15:29:12 +0100 Subject: [PATCH 5/5] fmt --- metricbeat/helper/elastic/elastic.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metricbeat/helper/elastic/elastic.go b/metricbeat/helper/elastic/elastic.go index 3000270f4f5e..e6cdea008f82 100644 --- a/metricbeat/helper/elastic/elastic.go +++ b/metricbeat/helper/elastic/elastic.go @@ -173,10 +173,10 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpa for _, cfgdMs := range cfgdMetricsetsStrings { found := false for _, ms := range optionalXpackMetricsets { - if ms == cfgdMs { - found = true - break - } + if ms == cfgdMs { + found = true + break + } } if found {