From 3ea9d103f68b6123d316919f5f57d0f7bc73df69 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jan 2020 17:23:00 -0800 Subject: [PATCH 1/5] Adding integration test for logstash module, xpack data path --- metricbeat/mb/testing/modules.go | 39 ++++++++++++++----- .../logstash/logstash_integration_test.go | 27 +++++++++++++ 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/metricbeat/mb/testing/modules.go b/metricbeat/mb/testing/modules.go index 12ff0e9a1add..9ada93f4f44d 100644 --- a/metricbeat/mb/testing/modules.go +++ b/metricbeat/mb/testing/modules.go @@ -86,6 +86,20 @@ func NewTestModule(t testing.TB, config interface{}) *TestModule { // The ModuleFactory and MetricSetFactory are obtained from the global // Registry. func NewMetricSet(t testing.TB, config interface{}) mb.MetricSet { + metricsets := NewMetricSets(t, config) + + if len(metricsets) != 1 { + t.Fatal("invalid number of metricsets instantiated") + } + + metricset := metricsets[0] + if metricset == nil { + t.Fatal("metricset is nil") + } + return metricset +} + +func NewMetricSets(t testing.TB, config interface{}) []mb.MetricSet { c, err := common.NewConfigFrom(config) if err != nil { t.Fatal(err) @@ -98,15 +112,7 @@ func NewMetricSet(t testing.TB, config interface{}) mb.MetricSet { t.Fatal("no module instantiated") } - if len(metricsets) != 1 { - t.Fatal("invalid number of metricsets instantiated") - } - - metricset := metricsets[0] - if metricset == nil { - t.Fatal("metricset is nil") - } - return metricset + return metricsets } // NewEventFetcher instantiates a new EventFetcher using the given @@ -182,6 +188,21 @@ func NewReportingMetricSetV2Error(t testing.TB, config interface{}) mb.Reporting return reportingMetricSetV2Error } +func NewReportingMetricSetV2Errors(t testing.TB, config interface{}) []mb.ReportingMetricSetV2Error { + metricSets := NewMetricSets(t, config) + var reportingMetricSets []mb.ReportingMetricSetV2Error + for _, metricSet := range metricSets { + rMS, ok := metricSet.(mb.ReportingMetricSetV2Error) + if !ok { + t.Fatalf("MetricSet %v does not implement ReportingMetricSetV2Error", metricSet.Name()) + } + + reportingMetricSets = append(reportingMetricSets, rMS) + } + + return reportingMetricSets +} + // NewReportingMetricSetV2WithContext returns a new ReportingMetricSetV2WithContext instance. Then // you can use ReportingFetchV2 to perform a Fetch operation with the MetricSet. func NewReportingMetricSetV2WithContext(t testing.TB, config interface{}) mb.ReportingMetricSetV2WithContext { diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 15eb637faf6d..37328ea16613 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -70,3 +70,30 @@ func TestData(t *testing.T) { }) } } + +func TestXPackEnabled(t *testing.T) { + service := compose.EnsureUp(t, "logstash") + nameToTypeMap := map[string]string{ + "node": "logstash_state", + "node_stats": "logstash_stats", + } + + config := map[string]interface{}{ + "module": "logstash", + "metricsets": metricSets, + "hosts": []string{"localhost:9600"}, + "xpack.enabled": true, + } + metricSets := mbtest.NewReportingMetricSetV2Errors(t, config) + for _, f := range metricSets { + events, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + + event := events[0] + assert.Equal(t, nameToTypeMap[f.Name()], event.RootFields["type"]) + assert.Regexp(t, `^.monitoring-logstash-\d-mb`, event.Index) + } +} From 525f378310cfbfe9306adf1ee055d7d26ee43c9b Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jan 2020 17:27:33 -0800 Subject: [PATCH 2/5] Adding godoc --- metricbeat/mb/testing/modules.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metricbeat/mb/testing/modules.go b/metricbeat/mb/testing/modules.go index 9ada93f4f44d..3edd1baeef9b 100644 --- a/metricbeat/mb/testing/modules.go +++ b/metricbeat/mb/testing/modules.go @@ -99,6 +99,8 @@ func NewMetricSet(t testing.TB, config interface{}) mb.MetricSet { return metricset } +// NewMetricSets instantiates a list of new MetricSets using the given +// module configuration. func NewMetricSets(t testing.TB, config interface{}) []mb.MetricSet { c, err := common.NewConfigFrom(config) if err != nil { @@ -188,6 +190,7 @@ func NewReportingMetricSetV2Error(t testing.TB, config interface{}) mb.Reporting return reportingMetricSetV2Error } +// NewReportingMetricSetV2Errors returns an array of new ReportingMetricSetV2 instances. func NewReportingMetricSetV2Errors(t testing.TB, config interface{}) []mb.ReportingMetricSetV2Error { metricSets := NewMetricSets(t, config) var reportingMetricSets []mb.ReportingMetricSetV2Error From f75beeff5edb0b3577ac2e81fbda4c404f12dea0 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jan 2020 17:32:45 -0800 Subject: [PATCH 3/5] Refactoring --- .../logstash/logstash_integration_test.go | 26 ++++++++++++------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 37328ea16613..d4164590f939 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -73,27 +73,33 @@ func TestData(t *testing.T) { func TestXPackEnabled(t *testing.T) { service := compose.EnsureUp(t, "logstash") - nameToTypeMap := map[string]string{ + + metricSetToTypeMap := map[string]string{ "node": "logstash_state", "node_stats": "logstash_stats", } - config := map[string]interface{}{ - "module": "logstash", - "metricsets": metricSets, - "hosts": []string{"localhost:9600"}, - "xpack.enabled": true, - } + config := getModuleXPackConfig(service.Host()) + metricSets := mbtest.NewReportingMetricSetV2Errors(t, config) - for _, f := range metricSets { - events, errs := mbtest.ReportingFetchV2Error(f) + for _, metricSet := range metricSets { + events, errs := mbtest.ReportingFetchV2Error(metricSet) assert.Empty(t, errs) if !assert.NotEmpty(t, events) { t.FailNow() } event := events[0] - assert.Equal(t, nameToTypeMap[f.Name()], event.RootFields["type"]) + assert.Equal(t, metricSetToTypeMap[metricSet.Name()], event.RootFields["type"]) assert.Regexp(t, `^.monitoring-logstash-\d-mb`, event.Index) } } + +func getModuleXPackConfig(host string) map[string]interface{} { + return map[string]interface{}{ + "module": logstash.ModuleName, + "metricsets": metricSets, + "hosts": []string{host}, + "xpack.enabled": true, + } +} From e0546bb75048ae43a712fc483e4317a70517bf1f Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 23 Jan 2020 17:34:53 -0800 Subject: [PATCH 4/5] More refactoring --- .../logstash/logstash_integration_test.go | 20 +++++++++----- metricbeat/module/logstash/testing.go | 27 ------------------- 2 files changed, 14 insertions(+), 33 deletions(-) delete mode 100644 metricbeat/module/logstash/testing.go diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index d4164590f939..37a29b262402 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -41,7 +41,7 @@ func TestFetch(t *testing.T) { for _, metricSet := range metricSets { t.Run(metricSet, func(t *testing.T) { - config := logstash.GetConfig(metricSet, service.Host()) + config := getConfig(metricSet, service.Host()) f := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(f) @@ -57,11 +57,11 @@ func TestFetch(t *testing.T) { } func TestData(t *testing.T) { - service := compose.EnsureUp(t, "logstash") + service := compose.EnsureUpWithTimeout(t, 300, "logstash") for _, metricSet := range metricSets { t.Run(metricSet, func(t *testing.T) { - config := logstash.GetConfig(metricSet, service.Host()) + config := getConfig(metricSet, service.Host()) f := mbtest.NewReportingMetricSetV2Error(t, config) err := mbtest.WriteEventsReporterV2Error(f, t, metricSet) if err != nil { @@ -72,14 +72,14 @@ func TestData(t *testing.T) { } func TestXPackEnabled(t *testing.T) { - service := compose.EnsureUp(t, "logstash") + service := compose.EnsureUpWithTimeout(t, 300, "logstash") metricSetToTypeMap := map[string]string{ "node": "logstash_state", "node_stats": "logstash_stats", } - config := getModuleXPackConfig(service.Host()) + config := getXPackConfig(service.Host()) metricSets := mbtest.NewReportingMetricSetV2Errors(t, config) for _, metricSet := range metricSets { @@ -95,7 +95,15 @@ func TestXPackEnabled(t *testing.T) { } } -func getModuleXPackConfig(host string) map[string]interface{} { +func getConfig(metricSet string, host string) map[string]interface{} { + return map[string]interface{}{ + "module": ModuleName, + "metricsets": []string{metricSet}, + "hosts": []string{host}, + } +} + +func getXPackConfig(host string) map[string]interface{} { return map[string]interface{}{ "module": logstash.ModuleName, "metricsets": metricSets, diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go deleted file mode 100644 index b63ce5cc5c25..000000000000 --- a/metricbeat/module/logstash/testing.go +++ /dev/null @@ -1,27 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package logstash - -// GetConfig for Logstash -func GetConfig(metricset string, host string) map[string]interface{} { - return map[string]interface{}{ - "module": ModuleName, - "metricsets": []string{metricset}, - "hosts": []string{host}, - } -} From ab6b37ac4bad8156cd619e6192b00f903a3a5ed6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Thu, 30 Jan 2020 10:05:29 -0800 Subject: [PATCH 5/5] Fixing reference --- metricbeat/module/logstash/logstash_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go index 37a29b262402..fe9c54e46d7a 100644 --- a/metricbeat/module/logstash/logstash_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -97,7 +97,7 @@ func TestXPackEnabled(t *testing.T) { func getConfig(metricSet string, host string) map[string]interface{} { return map[string]interface{}{ - "module": ModuleName, + "module": logstash.ModuleName, "metricsets": []string{metricSet}, "hosts": []string{host}, }