Add support for optional metricsets in xpack mode

metricbeat/helper/elastic/elastic.go

@@ -138,7 +138,7 @@ func FixTimestampField(m mapstr.M, field string) error {
 }
 
 // NewModule returns a new Elastic stack module with the appropriate metricsets configured.
-func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) {
+func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, optionalXpackMetricsets []string, logger *logp.Logger) (*mb.BaseModule, error) {
 	moduleName := base.Name()
 
 	config := struct {
@@ -158,8 +158,35 @@ func NewModule(base *mb.BaseModule, xpackEnabledMetricsets []string, logger *log
 		return nil, errors.Wrapf(err, "could not unpack configuration for module %v", moduleName)
 	}
 
-	// These metricsets are exactly the ones required if xpack.enabled == true
-	raw["metricsets"] = xpackEnabledMetricsets
+	// Ensure all required metricsets are enabled when xpack.enabled == true, and add any additional which are optional
+	cfgdMetricsets, err := raw.GetValue("metricsets")
+	metricsets := xpackEnabledMetricsets
+	if err == nil && cfgdMetricsets != nil {
+		// Type cast the metricsets to a slice of strings
+		cfgdMetricsetsSlice := cfgdMetricsets.([]interface{})
+		cfgdMetricsetsStrings := make([]string, len(cfgdMetricsetsSlice))
+		for i := range cfgdMetricsetsSlice {
+			cfgdMetricsetsStrings[i] = cfgdMetricsetsSlice[i].(string)
+		}
+
+		// Add any optional metricsets which are not already configured
+		for _, cfgdMs := range cfgdMetricsetsStrings {
+			found := false
+			for _, ms := range optionalXpackMetricsets {
+				if ms == cfgdMs {
+					found = true
+					break
+				}
+			}
+
+			if found {
+				metricsets = append(metricsets, cfgdMs)
+			}
+		}
+
+	}
+
+	raw["metricsets"] = metricsets
 
 	newConfig, err := conf.NewConfigFrom(raw)
 	if err != nil {

metricbeat/helper/elastic/elastic_test.go

@@ -157,18 +157,22 @@ func TestConfigureModule(t *testing.T) {
 	require.NoError(t, err)
 	err = mockRegistry.AddMetricSet(moduleName, "baz", mockMetricSetFactory)
 	require.NoError(t, err)
+	err = mockRegistry.AddMetricSet(moduleName, "thud", mockMetricSetFactory)
+	require.NoError(t, err)
 
 	tests := map[string]struct {
-		initConfig             metricSetConfig
-		xpackEnabledMetricsets []string
-		newConfig              metricSetConfig
+		initConfig              metricSetConfig
+		xpackEnabledMetricsets  []string
+		optionalXpackMetricsets []string
+		newConfig               metricSetConfig
 	}{
 		"no_xpack_enabled": {
 			metricSetConfig{
 				Module:     moduleName,
 				MetricSets: []string{"foo", "bar"},
 			},
 			[]string{"baz", "qux", "foo"},
+			[]string{},
 			metricSetConfig{
 				Module:     moduleName,
 				MetricSets: []string{"foo", "bar"},
@@ -181,12 +185,27 @@ func TestConfigureModule(t *testing.T) {
 				MetricSets:   []string{"foo", "bar"},
 			},
 			[]string{"baz", "qux", "foo"},
+			[]string{},
 			metricSetConfig{
 				Module:       moduleName,
 				XPackEnabled: true,
 				MetricSets:   []string{"baz", "qux", "foo"},
 			},
 		},
+		"xpack_enabled_with_optional": {
+			metricSetConfig{
+				Module:       moduleName,
+				XPackEnabled: true,
+				MetricSets:   []string{"foo", "bar", "thud"},
+			},
+			[]string{"baz", "qux", "foo"},
+			[]string{"bar"},
+			metricSetConfig{
+				Module:       moduleName,
+				XPackEnabled: true,
+				MetricSets:   []string{"baz", "qux", "foo", "bar"}, // include optional, exclude others
+			},
+		},
 	}
 
 	for name, test := range tests {
@@ -200,7 +219,7 @@ func TestConfigureModule(t *testing.T) {
 				require.Fail(t, "expecting module to be base module")
 			}
 
-			newM, err := NewModule(bm, test.xpackEnabledMetricsets, logp.L())
+			newM, err := NewModule(bm, test.xpackEnabledMetricsets, test.optionalXpackMetricsets, logp.L())
 			require.NoError(t, err)
 
 			var newConfig metricSetConfig

metricbeat/module/beat/beat.go

@@ -41,7 +41,7 @@ var metricSets = []string{"state", "stats"}
 
 // NewModule creates a new module
 func NewModule(base mb.BaseModule) (mb.Module, error) {
-	return elastic.NewModule(&base, metricSets, logp.NewLogger(ModuleName))
+	return elastic.NewModule(&base, metricSets, []string{}, logp.NewLogger(ModuleName))
 }
 
 // ModuleName is the name of this module.

metricbeat/module/elasticsearch/elasticsearch.go

@@ -56,7 +56,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 		"node_stats",
 		"shard",
 	}
-	return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))
+	return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName))
 }
 
 var (

metricbeat/module/kibana/kibana.go

@@ -92,7 +92,7 @@ func NewModule(base mb.BaseModule) (mb.Module, error) {
 	xpackEnabledMetricSets := []string{
 		"stats", "cluster_rules", "node_rules", "cluster_actions", "node_actions",
 	}
-	return elastic.NewModule(&base, xpackEnabledMetricSets, logp.NewLogger(ModuleName))
+	return elastic.NewModule(&base, xpackEnabledMetricSets, []string{}, logp.NewLogger(ModuleName))
 }
 
 // GetVersion returns the version of the Kibana instance

metricbeat/module/logstash/logstash.go

@@ -40,7 +40,7 @@ func init() {
 
 // NewModule creates a new module
 func NewModule(base mb.BaseModule) (mb.Module, error) {
-	return elastic.NewModule(&base, []string{"node", "node_stats"}, logp.NewLogger(ModuleName))
+	return elastic.NewModule(&base, []string{"node", "node_stats"}, []string{}, logp.NewLogger(ModuleName))
 }
 
 // ModuleName is the name of this module.