From f67d8d7df70e1bcd705aeda32672b1dbbe014227 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 17:32:18 +0200
Subject: [PATCH 01/88] Initial implementation of compose test runner

---
 libbeat/tests/compose/project.go              | 12 ---
 libbeat/tests/compose/runner.go               | 75 +++++++++++++++++++
 metricbeat/docker-compose.yml                 | 10 ++-
 .../keyspace/keyspace_integration_test.go     | 55 +++++++++-----
 4 files changed, 122 insertions(+), 30 deletions(-)
 create mode 100644 libbeat/tests/compose/runner.go

diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index 6df5d3585600..b19b7e75b992 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -94,18 +94,6 @@ func NewProject(name string, files []string) (*Project, error) {
 
 // Start the container, unless it's running already
 func (c *Project) Start(service string) error {
-	servicesStatus, err := c.getServices(service)
-	if err != nil {
-		return err
-	}
-
-	if servicesStatus[service] != nil {
-		if servicesStatus[service].Running {
-			// Someone is running it
-			return nil
-		}
-	}
-
 	c.Lock()
 	defer c.Unlock()
 
diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
new file mode 100644
index 000000000000..9641cde9c4a1
--- /dev/null
+++ b/libbeat/tests/compose/runner.go
@@ -0,0 +1,75 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compose
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"testing"
+)
+
+type TestRunner struct {
+	Service  string
+	Options  map[string][]string
+	Parallel bool
+}
+
+func (r *TestRunner) scenarios() []map[string]string {
+	n := 1
+	for _, values := range r.Options {
+		n *= len(values)
+	}
+
+	scenarios := make([]map[string]string, n)
+	for variable, values := range r.Options {
+		v := 0
+		for i, s := range scenarios {
+			if s == nil {
+				s = make(map[string]string)
+				scenarios[i] = s
+			}
+			s[variable] = values[v]
+			v = (v + 1) % len(values)
+		}
+	}
+
+	return scenarios
+}
+
+func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
+	for _, s := range r.scenarios() {
+		var vars []string
+		for k, v := range s {
+			os.Setenv(k, v)
+			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
+		}
+		sort.Strings(vars)
+		desc := strings.Join(vars, ",")
+
+		t.Run(desc, func(t *testing.T) {
+			EnsureUp(t, r.Service)
+			for _, test := range tests {
+				t.Run(desc, test)
+			}
+			// Down(t, r.Service)
+		})
+
+	}
+}
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 0ddc7bf6cd1d..5b009c5276ee 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,5 @@
-version: '2'
+version: "2.4"
+
 services:
   beat:
     build: ${PWD}/.
@@ -157,7 +158,14 @@ services:
     build: ./module/rabbitmq/_meta
 
   redis:
+    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
     build: ./module/redis/_meta
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 6379"
+      interval: 1s
+      retries: 90
 
   traefik:
     build: ./module/traefik/_meta
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 35710d174e20..a0975519363b 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -33,29 +33,50 @@ import (
 
 var host = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+var allVersions = []string{
+	"3.2.4",
+	"4.0.11",
+	"5.0-rc",
+}
 
-	addEntry(t)
+var allOS = []string{
+	"alpine",
+	"stretch",
+}
 
-	// Fetch data
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if err != nil {
-		t.Fatal("fetch", err)
-	}
+var runner = compose.TestRunner{
+	Service: "redis",
+	Options: map[string][]string{
+		"REDIS_VERSION": allVersions,
+		"IMAGE_OS":      allOS,
+	},
+	Parallel: true,
+}
+
+func TestFetch(t *testing.T) {
+	runner.Run(t, func(t *testing.T) {
+		addEntry(t)
+
+		// Fetch data
+		f := mbtest.NewEventsFetcher(t, getConfig())
+		events, err := f.Fetch()
+		if err != nil {
+			t.Fatal("fetch", err)
+		}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
 
-	// Make sure at least 1 db keyspace exists
-	assert.True(t, len(events) > 0)
+		// Make sure at least 1 db keyspace exists
+		assert.True(t, len(events) > 0)
 
-	keyspace := events[0]
+		keyspace := events[0]
 
-	assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-	assert.True(t, keyspace["expires"].(int64) >= 0)
-	assert.True(t, keyspace["keys"].(int64) >= 0)
-	assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+		assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+		assert.True(t, keyspace["expires"].(int64) >= 0)
+		assert.True(t, keyspace["keys"].(int64) >= 0)
+		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+	},
+	)
 }
 
 func TestData(t *testing.T) {

From 39cb4922c3feb2fb46deca32aa0e498052e16541 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 19:32:42 +0200
Subject: [PATCH 02/88] Parallel

---
 libbeat/tests/compose/compose.go              | 30 ++++++---
 libbeat/tests/compose/project.go              | 34 ++++++++++
 libbeat/tests/compose/runner.go               | 40 +++++++++--
 libbeat/tests/compose/wrapper.go              | 40 ++++++++++-
 metricbeat/docker-compose.yml                 | 12 +---
 metricbeat/module/redis/_meta/Dockerfile      |  2 -
 metricbeat/module/redis/docker-compose.yml    | 11 +++
 .../keyspace/keyspace_integration_test.go     | 67 ++++++++++---------
 8 files changed, 177 insertions(+), 59 deletions(-)
 delete mode 100644 metricbeat/module/redis/_meta/Dockerfile
 create mode 100644 metricbeat/module/redis/docker-compose.yml

diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go
index 9cd6dd205a8f..de1b19ebd8f8 100644
--- a/libbeat/tests/compose/compose.go
+++ b/libbeat/tests/compose/compose.go
@@ -40,7 +40,8 @@ func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
 		return
 	}
 
-	compose, err := getComposeProject()
+	name := os.Getenv("DOCKER_COMPOSE_PROJECT_NAME")
+	compose, err := getComposeProject(name)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,28 +66,37 @@ func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
 	}
 }
 
-func getComposeProject() (*Project, error) {
+func findComposePath() (string, error) {
 	// find docker-compose
 	path, err := os.Getwd()
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 	for {
 		if path == "/" {
-			return nil, errors.New("docker-compose.yml not found")
+			break
 		}
 
-		if _, err = os.Stat(filepath.Join(path, "docker-compose.yml")); err != nil {
-			path = filepath.Dir(path)
-		} else {
-			break
+		composePath := filepath.Join(path, "docker-compose.yml")
+		if _, err = os.Stat(composePath); err == nil {
+			return composePath, nil
 		}
+		path = filepath.Dir(path)
+	}
+
+	return "", errors.New("docker-compose.yml not found")
+}
+
+func getComposeProject(name string) (*Project, error) {
+	path, err := findComposePath()
+	if err != nil {
+		return nil, err
 	}
 
 	return NewProject(
-		os.Getenv("DOCKER_COMPOSE_PROJECT_NAME"),
+		name,
 		[]string{
-			filepath.Join(path, "docker-compose.yml"),
+			path,
 		},
 	)
 }
diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index b19b7e75b992..89249f88d08e 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -56,10 +56,13 @@ const (
 // Driver is the interface of docker compose implementations
 type Driver interface {
 	Up(ctx context.Context, opts UpOptions, service string) error
+	Down(ctx context.Context) error
 	Kill(ctx context.Context, signal string, service string) error
 	Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
 	// Containers(ctx context.Context, projectFilter Filter, filter ...string) ([]string, error)
 
+	SetParameters(map[string]string)
+
 	LockFile() string
 }
 
@@ -69,6 +72,7 @@ type ContainerStatus interface {
 	Healthy() bool
 	Running() bool
 	Old() bool
+	Host() string
 }
 
 // Project is a docker-compose project
@@ -137,6 +141,25 @@ func (c *Project) Wait(seconds int, services ...string) error {
 	return nil
 }
 
+// Host gets the host and port of a service
+func (c *Project) Host(service string) (string, error) {
+	servicesStatus, err := c.getServices(service)
+	if err != nil {
+		return "", err
+	}
+
+	if len(servicesStatus) == 0 {
+		return "", errors.New("no container running for service")
+	}
+
+	status, ok := servicesStatus[service]
+	if !ok || status.Host == "" {
+		return "", errors.New("unknown host:port for service")
+	}
+
+	return status.Host, nil
+}
+
 // Kill a container
 func (c *Project) Kill(service string) error {
 	c.Lock()
@@ -178,6 +201,14 @@ func (c *Project) KillOld(except []string) error {
 	return nil
 }
 
+// Down removes all resources of a project
+func (c *Project) Down() error {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.Driver.Down(context.Background())
+}
+
 // Lock acquires the lock (300s) timeout
 // Normally it should only be seconds that the lock is used, but in some cases it can take longer.
 func (c *Project) Lock() {
@@ -214,6 +245,8 @@ type serviceInfo struct {
 	Running bool
 	Healthy bool
 
+	Host string
+
 	// Has been up for too long?:
 	Old bool
 }
@@ -247,6 +280,7 @@ func (c *Project) getServices(filter ...string) (map[string]*serviceInfo, error)
 		if service.Healthy {
 			service.Old = c.Old()
 		}
+		service.Host = c.Host()
 		result[name] = service
 	}
 
diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 9641cde9c4a1..97da2db478df 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -18,6 +18,7 @@
 package compose
 
 import (
+	"crypto/sha1"
 	"fmt"
 	"os"
 	"sort"
@@ -29,6 +30,7 @@ type TestRunner struct {
 	Service  string
 	Options  map[string][]string
 	Parallel bool
+	Timeout  int
 }
 
 func (r *TestRunner) scenarios() []map[string]string {
@@ -53,7 +55,12 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
-func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
+func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string)) {
+	timeout := r.Timeout
+	if timeout == 0 {
+		timeout = 300
+	}
+
 	for _, s := range r.scenarios() {
 		var vars []string
 		for k, v := range s {
@@ -63,12 +70,37 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T)) {
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 
+		name := fmt.Sprintf("%s_%x", r.Service, sha1.Sum([]byte(desc)))
+		project, err := getComposeProject(name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		project.SetParameters(s)
+
 		t.Run(desc, func(t *testing.T) {
-			EnsureUp(t, r.Service)
+			if r.Parallel {
+				t.Parallel()
+			}
+
+			err := project.Start(r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer project.Down()
+
+			err = project.Wait(timeout, r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			host, err := project.Host(r.Service)
+			if err != nil {
+				t.Fatal(err)
+			}
+
 			for _, test := range tests {
-				t.Run(desc, test)
+				test(t, host)
 			}
-			// Down(t, r.Service)
 		})
 
 	}
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index f8ae629584a7..9657b66d10ba 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,8 +20,10 @@ package compose
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -39,6 +41,8 @@ const (
 type wrapperDriver struct {
 	Name  string
 	Files []string
+
+	Environment []string
 }
 
 type wrapperContainer struct {
@@ -61,8 +65,27 @@ func (c *wrapperContainer) Old() bool {
 	return strings.Contains(c.info.Status, "minute")
 }
 
+func (c *wrapperContainer) Host() string {
+	// TODO: Support multiple networks/ports
+	var ip string
+	for _, net := range c.info.NetworkSettings.Networks {
+		if len(net.IPAddress) > 0 {
+			ip = net.IPAddress
+			break
+		}
+	}
+	if len(ip) == 0 {
+		return ""
+	}
+
+	for _, port := range c.info.Ports {
+		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
+	}
+	return ""
+}
+
 func (d *wrapperDriver) LockFile() string {
-	return d.Files[0] + ".lock"
+	return d.Files[0] + "." + d.Name + ".lock"
 }
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
@@ -76,9 +99,20 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd := exec.CommandContext(ctx, "docker-compose", args...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
+	if len(d.Environment) > 0 {
+		cmd.Env = d.Environment
+	}
 	return cmd
 }
 
+func (d *wrapperDriver) SetParameters(params map[string]string) {
+	var env []string
+	for k, v := range params {
+		env = append(env, fmt.Sprintf("%s=%s", k, v))
+	}
+	d.Environment = env
+}
+
 func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string) error {
 	var args []string
 
@@ -99,6 +133,10 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 	return d.cmd(ctx, "up", args...).Run()
 }
 
+func (d *wrapperDriver) Down(ctx context.Context) error {
+	return d.cmd(ctx, "down", "-v").Run()
+}
+
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {
 	var args []string
 
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 5b009c5276ee..8f0205eb7566 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2"
 
 services:
   beat:
@@ -157,16 +157,6 @@ services:
   rabbitmq:
     build: ./module/rabbitmq/_meta
 
-  redis:
-    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
-    build: ./module/redis/_meta
-    healthcheck:
-      test:
-      - "CMD-SHELL"
-      - "nc -z localhost 6379"
-      interval: 1s
-      retries: 90
-
   traefik:
     build: ./module/traefik/_meta
 
diff --git a/metricbeat/module/redis/_meta/Dockerfile b/metricbeat/module/redis/_meta/Dockerfile
deleted file mode 100644
index 8f48048cc5fb..000000000000
--- a/metricbeat/module/redis/_meta/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM redis:3.2.4-alpine
-HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 6379
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
new file mode 100644
index 000000000000..8f1f92bede53
--- /dev/null
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -0,0 +1,11 @@
+version: "2.4"
+
+services:
+  redis:
+    image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 6379"
+      interval: 1s
+      retries: 90
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index a0975519363b..0f158f60deb4 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -25,40 +25,33 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/redis"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
 )
 
-var host = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
-
-var allVersions = []string{
-	"3.2.4",
-	"4.0.11",
-	"5.0-rc",
-}
-
-var allOS = []string{
-	"alpine",
-	"stretch",
-}
-
 var runner = compose.TestRunner{
 	Service: "redis",
 	Options: map[string][]string{
-		"REDIS_VERSION": allVersions,
-		"IMAGE_OS":      allOS,
+		"REDIS_VERSION": []string{
+			"3.2.4",
+			"4.0.11",
+			"5.0-rc",
+		},
+		"IMAGE_OS": []string{
+			"alpine",
+			"stretch",
+		},
 	},
 	Parallel: true,
 }
 
 func TestFetch(t *testing.T) {
-	runner.Run(t, func(t *testing.T) {
-		addEntry(t)
+	runner.Run(t, func(t *testing.T, host string) {
+		addEntry(t, host)
 
 		// Fetch data
-		f := mbtest.NewEventsFetcher(t, getConfig())
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
 		events, err := f.Fetch()
 		if err != nil {
 			t.Fatal("fetch", err)
@@ -75,25 +68,37 @@ func TestFetch(t *testing.T) {
 		assert.True(t, keyspace["expires"].(int64) >= 0)
 		assert.True(t, keyspace["keys"].(int64) >= 0)
 		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
-	},
-	)
+	})
 }
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "redis")
-
-	addEntry(t)
+	// TODO: Fix EnsureUp
+	runner := compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"4.0.11",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+			},
+		},
+		Parallel: true,
+	}
+	runner.Run(t, func(t *testing.T, host string) {
+		addEntry(t, host)
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
 
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	})
 }
 
 // addEntry adds an entry to redis
-func addEntry(t *testing.T) {
+func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
 	c, err := rd.Dial("tcp", host)
 	if err != nil {
@@ -106,7 +111,7 @@ func addEntry(t *testing.T) {
 	}
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},

From 1f12a3e6a3f44ffccae2f6a70666c87fe86b8c8b Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 13 Aug 2018 22:53:19 +0200
Subject: [PATCH 03/88] More parallel

---
 metricbeat/module/redis/keyspace/keyspace_integration_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 0f158f60deb4..736db48b0645 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -47,6 +47,8 @@ var runner = compose.TestRunner{
 }
 
 func TestFetch(t *testing.T) {
+	t.Parallel()
+
 	runner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
@@ -72,6 +74,8 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Parallel()
+
 	// TODO: Fix EnsureUp
 	runner := compose.TestRunner{
 		Service: "redis",

From 56729d7eedb669a1dc261834b1335b3145a726d4 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 18:53:25 +0200
Subject: [PATCH 04/88] --no-ansi for compose

---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 9657b66d10ba..2e67ef0f7851 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -90,7 +90,7 @@ func (d *wrapperDriver) LockFile() string {
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
 	var args []string
-	args = append(args, "--project-name", d.Name)
+	args = append(args, "--no-ansi", "--project-name", d.Name)
 	for _, f := range d.Files {
 		args = append(args, "--file", f)
 	}

From ce4ab513658a95518d2dcd4de43deed15ed879b9 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:27:00 +0200
Subject: [PATCH 05/88] Randomize project name

---
 libbeat/tests/compose/runner.go | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 97da2db478df..7a4a7e0a1467 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -18,12 +18,14 @@
 package compose
 
 import (
-	"crypto/sha1"
 	"fmt"
+	"math/rand"
 	"os"
 	"sort"
 	"strings"
 	"testing"
+
+	"github.com/pkg/errors"
 )
 
 type TestRunner struct {
@@ -70,7 +72,10 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 
-		name := fmt.Sprintf("%s_%x", r.Service, sha1.Sum([]byte(desc)))
+		seq := make([]byte, 16)
+		rand.Read(seq)
+		name := fmt.Sprintf("%s_%x", r.Service, seq)
+
 		project, err := getComposeProject(name)
 		if err != nil {
 			t.Fatal(err)
@@ -90,12 +95,12 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 
 			err = project.Wait(timeout, r.Service)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatal(errors.Wrapf(err, "waiting for %s/%s", r.Service, desc))
 			}
 
 			host, err := project.Host(r.Service)
 			if err != nil {
-				t.Fatal(err)
+				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
 			for _, test := range tests {

From c2b8060b879716c13daddba11b2b89018715495d Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:51:30 +0200
Subject: [PATCH 06/88] Seed random generator for compose project names

---
 libbeat/tests/compose/runner.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 7a4a7e0a1467..97cbcac1f79c 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -24,10 +24,15 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/pkg/errors"
 )
 
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
 type TestRunner struct {
 	Service  string
 	Options  map[string][]string

From 166709d165c5875897f2acf741a13ec784bf6626 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 19:51:53 +0200
Subject: [PATCH 07/88] Migrate info metricset

---
 .../redis/info/info_integration_test.go       | 104 +++++++++---------
 .../keyspace/keyspace_integration_test.go     |  36 +-----
 metricbeat/module/redis/mtest/runner.go       |  36 ++++++
 metricbeat/module/redis/testing.go            |  48 --------
 4 files changed, 94 insertions(+), 130 deletions(-)
 create mode 100644 metricbeat/module/redis/mtest/runner.go
 delete mode 100644 metricbeat/module/redis/testing.go

diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index bbcfd1b8601c..1f5ef18796ba 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -23,9 +23,8 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
-	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/redis"
+	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
@@ -35,70 +34,75 @@ const (
 	password = "foobared"
 )
 
-var redisHost = redis.GetRedisEnvHost() + ":" + redis.GetRedisEnvPort()
-
 func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+	t.Parallel()
 
-	f := mbtest.NewEventFetcher(t, getConfig(""))
-	event, err := f.Fetch()
-	if err != nil {
-		t.Fatal("fetch", err)
-	}
+	mtest.Runner.Run(t, func(t *testing.T, host string) {
+		f := mbtest.NewEventFetcher(t, getConfig("", host))
+		event, err := f.Fetch()
+		if err != nil {
+			t.Fatal("fetch", err)
+		}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	// Check fields
-	assert.Equal(t, 9, len(event))
-	server := event["server"].(common.MapStr)
-	assert.Equal(t, "standalone", server["mode"])
+		// Check fields
+		assert.Equal(t, 9, len(event))
+		server := event["server"].(common.MapStr)
+		assert.Equal(t, "standalone", server["mode"])
+	})
 }
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "redis")
+	t.Parallel()
 
-	f := mbtest.NewEventFetcher(t, getConfig(""))
+	// TODO: Fix EnsureUp for this kind of scenarios
+	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+		f := mbtest.NewEventFetcher(t, getConfig("", host))
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		err := mbtest.WriteEvent(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	})
 }
 
 func TestPasswords(t *testing.T) {
-	compose.EnsureUp(t, "redis")
-
-	// Add password and ensure it gets reset
-	defer func() {
-		err := resetPassword(redisHost, password)
+	t.Parallel()
+
+	mtest.Runner.Run(t, func(t *testing.T, redisHost string) {
+		// Add password and ensure it gets reset
+		defer func() {
+			err := resetPassword(redisHost, password)
+			if err != nil {
+				t.Fatal("resetting password", err)
+			}
+		}()
+
+		err := addPassword(redisHost, password)
 		if err != nil {
-			t.Fatal("resetting password", err)
+			t.Fatal("adding password", err)
 		}
-	}()
-
-	err := addPassword(redisHost, password)
-	if err != nil {
-		t.Fatal("adding password", err)
-	}
 
-	// Test Fetch metrics with missing password
-	f := mbtest.NewEventFetcher(t, getConfig(""))
-	_, err = f.Fetch()
-	if assert.Error(t, err, "missing password") {
-		assert.Contains(t, err, "NOAUTH Authentication required.")
-	}
+		// Test Fetch metrics with missing password
+		f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+		_, err = f.Fetch()
+		if assert.Error(t, err, "missing password") {
+			assert.Contains(t, err, "NOAUTH Authentication required.")
+		}
 
-	// Config redis and metricset with an invalid password
-	f = mbtest.NewEventFetcher(t, getConfig("blah"))
-	_, err = f.Fetch()
-	if assert.Error(t, err, "invalid password") {
-		assert.Contains(t, err, "ERR invalid password")
-	}
+		// Config redis and metricset with an invalid password
+		f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+		_, err = f.Fetch()
+		if assert.Error(t, err, "invalid password") {
+			assert.Contains(t, err, "ERR invalid password")
+		}
 
-	// Config redis and metricset with a valid password
-	f = mbtest.NewEventFetcher(t, getConfig(password))
-	_, err = f.Fetch()
-	assert.NoError(t, err, "valid password")
+		// Config redis and metricset with a valid password
+		f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+		_, err = f.Fetch()
+		assert.NoError(t, err, "valid password")
+	})
 }
 
 // addPassword will add a password to redis.
@@ -142,7 +146,7 @@ func writeToRedis(host string) error {
 	return err
 }
 
-func getConfig(password string) map[string]interface{} {
+func getConfig(password, redisHost string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 736db48b0645..a9b2d0992414 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -23,33 +23,17 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
 	rd "github.com/garyburd/redigo/redis"
 	"github.com/stretchr/testify/assert"
 )
 
-var runner = compose.TestRunner{
-	Service: "redis",
-	Options: map[string][]string{
-		"REDIS_VERSION": []string{
-			"3.2.4",
-			"4.0.11",
-			"5.0-rc",
-		},
-		"IMAGE_OS": []string{
-			"alpine",
-			"stretch",
-		},
-	},
-	Parallel: true,
-}
-
 func TestFetch(t *testing.T) {
 	t.Parallel()
 
-	runner.Run(t, func(t *testing.T, host string) {
+	mtest.Runner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		// Fetch data
@@ -76,20 +60,8 @@ func TestFetch(t *testing.T) {
 func TestData(t *testing.T) {
 	t.Parallel()
 
-	// TODO: Fix EnsureUp
-	runner := compose.TestRunner{
-		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
-				"4.0.11",
-			},
-			"IMAGE_OS": []string{
-				"alpine",
-			},
-		},
-		Parallel: true,
-	}
-	runner.Run(t, func(t *testing.T, host string) {
+	// TODO: Fix EnsureUp for this kind of scenarios
+	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		f := mbtest.NewEventsFetcher(t, getConfig(host))
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
new file mode 100644
index 000000000000..20b21e0dc20b
--- /dev/null
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -0,0 +1,36 @@
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"3.2.4",
+				"4.0.11",
+				"5.0-rc",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+				"stretch",
+			},
+		},
+		Parallel: true,
+	}
+
+	DataRunner = compose.TestRunner{
+		Service: "redis",
+		Options: map[string][]string{
+			"REDIS_VERSION": []string{
+				"4.0.11",
+			},
+			"IMAGE_OS": []string{
+				"alpine",
+			},
+		},
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/redis/testing.go b/metricbeat/module/redis/testing.go
deleted file mode 100644
index 3149a3043b2d..000000000000
--- a/metricbeat/module/redis/testing.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package redis
-
-import (
-	"os"
-)
-
-// Helper functions for testing used in the redis metricsets
-
-// GetRedisEnvHost returns the hostname of the Redis server to use for testing.
-// It reads the value from the REDIS_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetRedisEnvHost() string {
-	host := os.Getenv("REDIS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetRedisEnvPort returns the port of the Redis server to use for testing.
-// It reads the value from the REDIS_PORT environment variable and returns
-// 6379 if it is not set.
-func GetRedisEnvPort() string {
-	port := os.Getenv("REDIS_PORT")
-
-	if len(port) == 0 {
-		port = "6379"
-	}
-	return port
-}

From 0f1a8d6bbb6473efa2d94d47c0dbb5a69deb12bd Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 14 Aug 2018 20:10:18 +0200
Subject: [PATCH 08/88] Run test suites

---
 libbeat/tests/compose/runner.go               |   8 +-
 .../redis/info/info_integration_test.go       | 102 +++++++++---------
 .../keyspace/keyspace_integration_test.go     |  41 +++----
 3 files changed, 77 insertions(+), 74 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 97cbcac1f79c..cb291cbb3214 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -40,6 +40,8 @@ type TestRunner struct {
 	Timeout  int
 }
 
+type Suite map[string]func(t *testing.T, host string)
+
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	for _, values := range r.Options {
@@ -62,7 +64,7 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
-func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string)) {
+func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	timeout := r.Timeout
 	if timeout == 0 {
 		timeout = 300
@@ -108,8 +110,8 @@ func (r *TestRunner) Run(t *testing.T, tests ...func(t *testing.T, host string))
 				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
-			for _, test := range tests {
-				test(t, host)
+			for name, test := range tests {
+				t.Run(name, func(t *testing.T) { test(t, host) })
 			}
 		})
 
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 1f5ef18796ba..356aacc910aa 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -23,6 +23,7 @@ import (
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
@@ -34,22 +35,57 @@ const (
 	password = "foobared"
 )
 
-func TestFetch(t *testing.T) {
+func TestInfo(t *testing.T) {
 	t.Parallel()
 
-	mtest.Runner.Run(t, func(t *testing.T, host string) {
-		f := mbtest.NewEventFetcher(t, getConfig("", host))
-		event, err := f.Fetch()
-		if err != nil {
-			t.Fatal("fetch", err)
-		}
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, host string) {
+			f := mbtest.NewEventFetcher(t, getConfig("", host))
+			event, err := f.Fetch()
+			if err != nil {
+				t.Fatal("fetch", err)
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check fields
+			assert.Equal(t, 9, len(event))
+			server := event["server"].(common.MapStr)
+			assert.Equal(t, "standalone", server["mode"])
+		},
+		"Passwords": func(t *testing.T, redisHost string) {
+			// Add password and ensure it gets reset
+			defer func() {
+				err := resetPassword(redisHost, password)
+				if err != nil {
+					t.Fatal("resetting password", err)
+				}
+			}()
+
+			err := addPassword(redisHost, password)
+			if err != nil {
+				t.Fatal("adding password", err)
+			}
+
+			// Test Fetch metrics with missing password
+			f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+			_, err = f.Fetch()
+			if assert.Error(t, err, "missing password") {
+				assert.Contains(t, err, "NOAUTH Authentication required.")
+			}
 
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Config redis and metricset with an invalid password
+			f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+			_, err = f.Fetch()
+			if assert.Error(t, err, "invalid password") {
+				assert.Contains(t, err, "ERR invalid password")
+			}
 
-		// Check fields
-		assert.Equal(t, 9, len(event))
-		server := event["server"].(common.MapStr)
-		assert.Equal(t, "standalone", server["mode"])
+			// Config redis and metricset with a valid password
+			f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+			_, err = f.Fetch()
+			assert.NoError(t, err, "valid password")
+		},
 	})
 }
 
@@ -57,52 +93,14 @@ func TestData(t *testing.T) {
 	t.Parallel()
 
 	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
 		f := mbtest.NewEventFetcher(t, getConfig("", host))
 
 		err := mbtest.WriteEvent(f, t)
 		if err != nil {
 			t.Fatal("write", err)
 		}
-	})
-}
-
-func TestPasswords(t *testing.T) {
-	t.Parallel()
-
-	mtest.Runner.Run(t, func(t *testing.T, redisHost string) {
-		// Add password and ensure it gets reset
-		defer func() {
-			err := resetPassword(redisHost, password)
-			if err != nil {
-				t.Fatal("resetting password", err)
-			}
-		}()
-
-		err := addPassword(redisHost, password)
-		if err != nil {
-			t.Fatal("adding password", err)
-		}
-
-		// Test Fetch metrics with missing password
-		f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
-		_, err = f.Fetch()
-		if assert.Error(t, err, "missing password") {
-			assert.Contains(t, err, "NOAUTH Authentication required.")
-		}
-
-		// Config redis and metricset with an invalid password
-		f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
-		_, err = f.Fetch()
-		if assert.Error(t, err, "invalid password") {
-			assert.Contains(t, err, "ERR invalid password")
-		}
-
-		// Config redis and metricset with a valid password
-		f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
-		_, err = f.Fetch()
-		assert.NoError(t, err, "valid password")
-	})
+	}})
 }
 
 // addPassword will add a password to redis.
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index a9b2d0992414..be4d5cfc4176 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -23,6 +23,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/redis/mtest"
 
@@ -30,30 +31,32 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
+func TestKeyspace(t *testing.T) {
 	t.Parallel()
 
-	mtest.Runner.Run(t, func(t *testing.T, host string) {
-		addEntry(t, host)
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, host string) {
+			addEntry(t, host)
 
-		// Fetch data
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
-		events, err := f.Fetch()
-		if err != nil {
-			t.Fatal("fetch", err)
-		}
+			// Fetch data
+			f := mbtest.NewEventsFetcher(t, getConfig(host))
+			events, err := f.Fetch()
+			if err != nil {
+				t.Fatal("fetch", err)
+			}
 
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
 
-		// Make sure at least 1 db keyspace exists
-		assert.True(t, len(events) > 0)
+			// Make sure at least 1 db keyspace exists
+			assert.True(t, len(events) > 0)
 
-		keyspace := events[0]
+			keyspace := events[0]
 
-		assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-		assert.True(t, keyspace["expires"].(int64) >= 0)
-		assert.True(t, keyspace["keys"].(int64) >= 0)
-		assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+			assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+			assert.True(t, keyspace["expires"].(int64) >= 0)
+			assert.True(t, keyspace["keys"].(int64) >= 0)
+			assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+		},
 	})
 }
 
@@ -61,7 +64,7 @@ func TestData(t *testing.T) {
 	t.Parallel()
 
 	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, func(t *testing.T, host string) {
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
 		addEntry(t, host)
 
 		f := mbtest.NewEventsFetcher(t, getConfig(host))
@@ -70,7 +73,7 @@ func TestData(t *testing.T) {
 		if err != nil {
 			t.Fatal("write", err)
 		}
-	})
+	}})
 }
 
 // addEntry adds an entry to redis

From 89f5f0e94705e392683921f14f2340586037dbec Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 16:58:06 +0200
Subject: [PATCH 09/88] Allow to override compose runner host and options

Add the possibility to override some options of compose runner using
environment variables:
- If there is a variable with the upercased name of the service + "_HOST"
  (i.e. REDIS_HOST for "redis" service), no scenario is created and the
  value of the variable is injected to the test in the host variable.
- If there is a variable named as one of the options, this single value
  is used instead of the list of values in the options matrix.
---
 libbeat/tests/compose/runner.go | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index cb291cbb3214..be8fac9fab04 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -44,12 +44,19 @@ type Suite map[string]func(t *testing.T, host string)
 
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
-	for _, values := range r.Options {
+	options := make(map[string][]string)
+	for env, values := range r.Options {
+		// Allow to ovverride options from environment variables
+		value := os.Getenv(env)
+		if value != "" {
+			values = []string{value}
+		}
+		options[env] = values
 		n *= len(values)
 	}
 
 	scenarios := make([]map[string]string, n)
-	for variable, values := range r.Options {
+	for variable, values := range options {
 		v := 0
 		for i, s := range scenarios {
 			if s == nil {
@@ -64,7 +71,26 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
+func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
+	env := strings.ToUpper(r.Service) + "_HOST"
+	host := os.Getenv(env)
+	if host == "" {
+		return false
+	}
+
+	t.Logf("Test host overriden by %s=%s", env, host)
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) { test(t, host) })
+	}
+	return true
+}
+
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
+	if r.runHostOverride(t, tests) {
+		return
+	}
+
 	timeout := r.Timeout
 	if timeout == 0 {
 		timeout = 300

From 57e11ea50aad13131a4dcf00e477bd47192f0e85 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 17:08:59 +0200
Subject: [PATCH 10/88] Typo

---
 libbeat/tests/compose/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index be8fac9fab04..4b8c9912f1dc 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -46,7 +46,7 @@ func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	options := make(map[string][]string)
 	for env, values := range r.Options {
-		// Allow to ovverride options from environment variables
+		// Allow to override options from environment variables
 		value := os.Getenv(env)
 		if value != "" {
 			values = []string{value}

From d7fe3436a038981d91ace66f347f535a6546c55c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 15 Aug 2018 17:51:25 +0200
Subject: [PATCH 11/88] Use more explicit healthcheck for redis

---
 metricbeat/module/redis/docker-compose.yml | 2 +-
 metricbeat/module/redis/mtest/runner.go    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index 8f1f92bede53..f8e98e466851 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -6,6 +6,6 @@ services:
     healthcheck:
       test:
       - "CMD-SHELL"
-      - "nc -z localhost 6379"
+      - "redis-cli ping"
       interval: 1s
       retries: 90
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 20b21e0dc20b..343066c8ab0a 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -9,7 +9,7 @@ var (
 		Service: "redis",
 		Options: map[string][]string{
 			"REDIS_VERSION": []string{
-				"3.2.4",
+				"3.2.12",
 				"4.0.11",
 				"5.0-rc",
 			},

From d71d2101890f11dd45cd32cf275a47bd89754dc8 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 17 Aug 2018 18:35:02 +0200
Subject: [PATCH 12/88] Keep a redis service in the main docker-compose file

---
 metricbeat/docker-compose.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 8f0205eb7566..e8ad49a15278 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2"
+version: "2.4"
 
 services:
   beat:
@@ -157,6 +157,11 @@ services:
   rabbitmq:
     build: ./module/rabbitmq/_meta
 
+  redis:
+    extends:
+      file: ./module/redis/docker-compose.yml
+      service: redis
+
   traefik:
     build: ./module/traefik/_meta
 

From 938ba3258f7fa16c7603856066e405ab891a0757 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:14:23 +0200
Subject: [PATCH 13/88] Redis test use compose provided hosts

---
 metricbeat/tests/system/test_redis.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 0f2219feecb0..94b106ab929c 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -58,7 +58,10 @@ def test_keyspace(self):
         """
 
         # At least one event must be inserted so db stats exist
-        r = redis.StrictRedis(host=os.getenv('REDIS_HOST', 'localhost'), port=os.getenv('REDIS_PORT', '6379'), db=0)
+        r = redis.StrictRedis(
+            host=os.getenv('REDIS_HOST', self.compose_hosts()[0]),
+            port=os.getenv('REDIS_PORT', '6379'),
+            db=0)
         r.set('foo', 'bar')
 
         self.render_config_template(modules=[{
@@ -116,5 +119,5 @@ def test_module_processors(self):
         self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
 
     def get_hosts(self):
-        return [os.getenv('REDIS_HOST', 'localhost') + ':' +
+        return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]

From 4301a8d1ffbbbb291987dd55dc92cc8189295c01 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:42:22 +0200
Subject: [PATCH 14/88] Run beat container in host network mode

---
 metricbeat/docker-compose.yml                 | 30 +------------------
 metricbeat/module/aerospike/_meta/env         |  2 --
 metricbeat/module/apache/_meta/env            |  3 --
 metricbeat/module/ceph/_meta/env              |  2 --
 metricbeat/module/couchbase/_meta/env         |  3 --
 metricbeat/module/dropwizard/_meta/env        |  2 --
 metricbeat/module/elasticsearch/_meta/env     |  2 --
 .../elasticsearch/test_elasticsearch.py       |  2 +-
 metricbeat/module/envoyproxy/_meta/env        |  2 --
 metricbeat/module/etcd/_meta/env              |  2 --
 metricbeat/module/haproxy/_meta/env           |  4 ---
 metricbeat/module/http/_meta/env              |  2 --
 metricbeat/module/jolokia/_meta/env           |  2 --
 metricbeat/module/kafka/_meta/env             |  3 --
 metricbeat/module/kibana/_meta/env            |  2 --
 metricbeat/module/kubernetes/_meta/env        |  4 ---
 metricbeat/module/logstash/_meta/env          |  2 --
 metricbeat/module/memcached/_meta/env         |  2 --
 metricbeat/module/mongodb/_meta/env           |  2 --
 metricbeat/module/munin/_meta/env             |  2 --
 metricbeat/module/mysql/_meta/env             |  3 --
 metricbeat/module/nginx/_meta/env             |  2 --
 metricbeat/module/php_fpm/_meta/env           |  2 --
 metricbeat/module/postgresql/_meta/env        |  4 ---
 metricbeat/module/prometheus/_meta/env        |  2 --
 metricbeat/module/rabbitmq/_meta/env          |  4 ---
 metricbeat/module/redis/_meta/env             |  2 --
 metricbeat/module/traefik/_meta/env           |  2 --
 metricbeat/module/traefik/test_traefik.py     |  2 +-
 metricbeat/module/uwsgi/_meta/env             |  2 --
 metricbeat/module/zookeeper/_meta/env         |  2 --
 metricbeat/tests/system/test_aerospike.py     |  2 +-
 metricbeat/tests/system/test_apache.py        |  4 +--
 metricbeat/tests/system/test_ceph.py          |  2 +-
 metricbeat/tests/system/test_config.py        |  2 +-
 metricbeat/tests/system/test_couchbase.py     |  2 +-
 metricbeat/tests/system/test_dropwizard.py    |  2 +-
 metricbeat/tests/system/test_envoyproxy.py    |  2 +-
 metricbeat/tests/system/test_golang.py        |  2 +-
 metricbeat/tests/system/test_http.py          |  3 +-
 metricbeat/tests/system/test_jolokia.py       |  2 +-
 metricbeat/tests/system/test_kibana.py        |  2 +-
 metricbeat/tests/system/test_kubernetes.py    |  4 +--
 metricbeat/tests/system/test_logstash.py      |  2 +-
 metricbeat/tests/system/test_memcached.py     |  2 +-
 metricbeat/tests/system/test_mongodb.py       |  2 +-
 metricbeat/tests/system/test_munin.py         |  2 +-
 metricbeat/tests/system/test_mysql.py         |  4 ++-
 metricbeat/tests/system/test_phpfpm.py        |  2 +-
 metricbeat/tests/system/test_postgresql.py    | 11 +++++--
 metricbeat/tests/system/test_prometheus.py    |  2 +-
 metricbeat/tests/system/test_zookeeper.py     |  2 +-
 52 files changed, 37 insertions(+), 123 deletions(-)
 delete mode 100644 metricbeat/module/aerospike/_meta/env
 delete mode 100644 metricbeat/module/apache/_meta/env
 delete mode 100644 metricbeat/module/ceph/_meta/env
 delete mode 100644 metricbeat/module/couchbase/_meta/env
 delete mode 100644 metricbeat/module/dropwizard/_meta/env
 delete mode 100644 metricbeat/module/elasticsearch/_meta/env
 delete mode 100644 metricbeat/module/envoyproxy/_meta/env
 delete mode 100644 metricbeat/module/etcd/_meta/env
 delete mode 100644 metricbeat/module/haproxy/_meta/env
 delete mode 100644 metricbeat/module/http/_meta/env
 delete mode 100644 metricbeat/module/jolokia/_meta/env
 delete mode 100644 metricbeat/module/kafka/_meta/env
 delete mode 100644 metricbeat/module/kibana/_meta/env
 delete mode 100644 metricbeat/module/kubernetes/_meta/env
 delete mode 100644 metricbeat/module/logstash/_meta/env
 delete mode 100644 metricbeat/module/memcached/_meta/env
 delete mode 100644 metricbeat/module/mongodb/_meta/env
 delete mode 100644 metricbeat/module/munin/_meta/env
 delete mode 100644 metricbeat/module/mysql/_meta/env
 delete mode 100644 metricbeat/module/nginx/_meta/env
 delete mode 100644 metricbeat/module/php_fpm/_meta/env
 delete mode 100644 metricbeat/module/postgresql/_meta/env
 delete mode 100644 metricbeat/module/prometheus/_meta/env
 delete mode 100644 metricbeat/module/rabbitmq/_meta/env
 delete mode 100644 metricbeat/module/redis/_meta/env
 delete mode 100644 metricbeat/module/traefik/_meta/env
 delete mode 100644 metricbeat/module/uwsgi/_meta/env
 delete mode 100644 metricbeat/module/zookeeper/_meta/env

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e8ad49a15278..30ac09530b77 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -10,36 +10,8 @@ services:
       - ${PWD}/..:/go/src/github.com/elastic/beats/
       # This is required to on-demand launching the rest on containers for tests & also docker module tests:
       - /var/run/docker.sock:/var/run/docker.sock
+    network_mode: host
     command: make
-    env_file:
-      - ./module/aerospike/_meta/env
-      - ./module/apache/_meta/env
-      - ./module/ceph/_meta/env
-      - ./module/couchbase/_meta/env
-      - ./module/dropwizard/_meta/env
-      - ./module/elasticsearch/_meta/env
-      - ./module/envoyproxy/_meta/env
-      - ./module/etcd/_meta/env
-      - ./module/haproxy/_meta/env
-      - ./module/http/_meta/env
-      - ./module/jolokia/_meta/env
-      - ./module/kafka/_meta/env
-      - ./module/kibana/_meta/env
-      - ./module/kubernetes/_meta/env
-      - ./module/logstash/_meta/env
-      - ./module/memcached/_meta/env
-      - ./module/mongodb/_meta/env
-      - ./module/munin/_meta/env
-      - ./module/mysql/_meta/env
-      - ./module/nginx/_meta/env
-      - ./module/php_fpm/_meta/env
-      - ./module/postgresql/_meta/env
-      - ./module/prometheus/_meta/env
-      - ./module/rabbitmq/_meta/env
-      - ./module/redis/_meta/env
-      - ./module/traefik/_meta/env
-      - ./module/uwsgi/_meta/env
-      - ./module/zookeeper/_meta/env
 
   # Modules
   aerospike:
diff --git a/metricbeat/module/aerospike/_meta/env b/metricbeat/module/aerospike/_meta/env
deleted file mode 100644
index 2c95ea957cb2..000000000000
--- a/metricbeat/module/aerospike/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-AEROSPIKE_HOST=aerospike
-AEROSPIKE_PORT=3000
diff --git a/metricbeat/module/apache/_meta/env b/metricbeat/module/apache/_meta/env
deleted file mode 100644
index 5339c3407ed8..000000000000
--- a/metricbeat/module/apache/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-APACHE_HOST=apache
-APACHE_OLD_HOST=apache_2_4_12
-APACHE_PORT=80
diff --git a/metricbeat/module/ceph/_meta/env b/metricbeat/module/ceph/_meta/env
deleted file mode 100644
index 27b7ce7dbfb3..000000000000
--- a/metricbeat/module/ceph/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-CEPH_HOST=ceph
-CEPH_PORT=5000
diff --git a/metricbeat/module/couchbase/_meta/env b/metricbeat/module/couchbase/_meta/env
deleted file mode 100644
index 38aa8e347786..000000000000
--- a/metricbeat/module/couchbase/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-COUCHBASE_HOST=couchbase
-COUCHBASE_PORT=8091
-COUCHBASE_DSN=http://Administrator:password@couchbase:8091
diff --git a/metricbeat/module/dropwizard/_meta/env b/metricbeat/module/dropwizard/_meta/env
deleted file mode 100644
index 771807862fd8..000000000000
--- a/metricbeat/module/dropwizard/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-DROPWIZARD_HOST=dropwizard
-DROPWIZARD_PORT=8080
diff --git a/metricbeat/module/elasticsearch/_meta/env b/metricbeat/module/elasticsearch/_meta/env
deleted file mode 100644
index 5856225ceaf9..000000000000
--- a/metricbeat/module/elasticsearch/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ES_HOST=elasticsearch
-ES_PORT=9200
diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py
index 90906ac641dd..49cdb4d4a4ed 100644
--- a/metricbeat/module/elasticsearch/test_elasticsearch.py
+++ b/metricbeat/module/elasticsearch/test_elasticsearch.py
@@ -39,7 +39,7 @@ def test_metricsets(self, metricset):
                              ["service.name"], extras={"index_recovery.active_only": "false"})
 
     def get_hosts(self):
-        return [os.getenv('ES_HOST', 'localhost') + ':' +
+        return [os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ES_PORT', '9200')]
 
     def create_ml_job(self):
diff --git a/metricbeat/module/envoyproxy/_meta/env b/metricbeat/module/envoyproxy/_meta/env
deleted file mode 100644
index 75a2386e7f6e..000000000000
--- a/metricbeat/module/envoyproxy/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ENVOYPROXY_HOST=envoyproxy
-ENVOYPROXY_PORT=9901
diff --git a/metricbeat/module/etcd/_meta/env b/metricbeat/module/etcd/_meta/env
deleted file mode 100644
index ea9d54409853..000000000000
--- a/metricbeat/module/etcd/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ETCD_HOST=etcd
-ETCD_PORT=2379
diff --git a/metricbeat/module/haproxy/_meta/env b/metricbeat/module/haproxy/_meta/env
deleted file mode 100644
index c0762e53c17b..000000000000
--- a/metricbeat/module/haproxy/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-HAPROXY_1_6_HOST=haproxy_1_6
-HAPROXY_1_7_HOST=haproxy_1_7
-HAPROXY_HOST=haproxy
-HAPROXY_PORT=14567
diff --git a/metricbeat/module/http/_meta/env b/metricbeat/module/http/_meta/env
deleted file mode 100644
index dfe04431ecac..000000000000
--- a/metricbeat/module/http/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-HTTP_HOST=http
-HTTP_PORT=8080
diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env
deleted file mode 100644
index 9c0340b6f3c5..000000000000
--- a/metricbeat/module/jolokia/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-JOLOKIA_HOST=jolokia
-JOLOKIA_PORT=8778
diff --git a/metricbeat/module/kafka/_meta/env b/metricbeat/module/kafka/_meta/env
deleted file mode 100644
index 227657f404e3..000000000000
--- a/metricbeat/module/kafka/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-KAFKA_0_10_2_HOST=kafka_0_10_2
-KAFKA_HOST=kafka
-KAFKA_PORT=9092
diff --git a/metricbeat/module/kibana/_meta/env b/metricbeat/module/kibana/_meta/env
deleted file mode 100644
index a22fc93ec709..000000000000
--- a/metricbeat/module/kibana/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-KIBANA_HOST=kibana
-KIBANA_PORT=5601
\ No newline at end of file
diff --git a/metricbeat/module/kubernetes/_meta/env b/metricbeat/module/kubernetes/_meta/env
deleted file mode 100644
index 03cff050efa3..000000000000
--- a/metricbeat/module/kubernetes/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-KUBELET_HOST=172.17.0.1
-KUBELET_PORT=10255
-KUBE_STATE_METRICS_HOST=kubestate
-KUBE_STATE_METRICS_PORT=8080
diff --git a/metricbeat/module/logstash/_meta/env b/metricbeat/module/logstash/_meta/env
deleted file mode 100644
index 57f4dc268f76..000000000000
--- a/metricbeat/module/logstash/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-LOGSTASH_HOST=logstash
-LOGSTASH_PORT=9600
diff --git a/metricbeat/module/memcached/_meta/env b/metricbeat/module/memcached/_meta/env
deleted file mode 100644
index 3efa58cda165..000000000000
--- a/metricbeat/module/memcached/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MEMCACHED_HOST=memcached
-MEMCACHED_PORT=11211
diff --git a/metricbeat/module/mongodb/_meta/env b/metricbeat/module/mongodb/_meta/env
deleted file mode 100644
index a51807cbe81e..000000000000
--- a/metricbeat/module/mongodb/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MONGODB_HOST=mongodb
-MONGODB_PORT=27017
diff --git a/metricbeat/module/munin/_meta/env b/metricbeat/module/munin/_meta/env
deleted file mode 100644
index b81c5ee86790..000000000000
--- a/metricbeat/module/munin/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-MUNIN_HOST=munin
-MUNIN_PORT=4949
diff --git a/metricbeat/module/mysql/_meta/env b/metricbeat/module/mysql/_meta/env
deleted file mode 100644
index e6aa44eaf4a6..000000000000
--- a/metricbeat/module/mysql/_meta/env
+++ /dev/null
@@ -1,3 +0,0 @@
-MYSQL_DSN=root:test@tcp(mysql:3306)/
-MYSQL_HOST=mysql
-MYSQL_PORT=3306
diff --git a/metricbeat/module/nginx/_meta/env b/metricbeat/module/nginx/_meta/env
deleted file mode 100644
index 693e4c78989b..000000000000
--- a/metricbeat/module/nginx/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-NGINX_HOST=nginx
-NGINX_PORT=80
diff --git a/metricbeat/module/php_fpm/_meta/env b/metricbeat/module/php_fpm/_meta/env
deleted file mode 100644
index 3437fa5b8c42..000000000000
--- a/metricbeat/module/php_fpm/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-PHPFPM_HOST=phpfpm
-PHPFPM_PORT=81
diff --git a/metricbeat/module/postgresql/_meta/env b/metricbeat/module/postgresql/_meta/env
deleted file mode 100644
index eef1c64138ce..000000000000
--- a/metricbeat/module/postgresql/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
-POSTGRESQL_HOST=postgresql
-POSTGRESQL_PORT=5432
-POSTGRESQL_USERNAME=postgres
diff --git a/metricbeat/module/prometheus/_meta/env b/metricbeat/module/prometheus/_meta/env
deleted file mode 100644
index a77775f7c6f1..000000000000
--- a/metricbeat/module/prometheus/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-PROMETHEUS_HOST=prometheus
-PROMETHEUS_PORT=9090
diff --git a/metricbeat/module/rabbitmq/_meta/env b/metricbeat/module/rabbitmq/_meta/env
deleted file mode 100644
index a46bb9673d88..000000000000
--- a/metricbeat/module/rabbitmq/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_PORT=15672
-RABBITMQ_USERNAME=guest
-RABBITMQ_PASSWORD=guest
diff --git a/metricbeat/module/redis/_meta/env b/metricbeat/module/redis/_meta/env
deleted file mode 100644
index 82b68167bc63..000000000000
--- a/metricbeat/module/redis/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-REDIS_HOST=redis
-REDIS_PORT=6379
diff --git a/metricbeat/module/traefik/_meta/env b/metricbeat/module/traefik/_meta/env
deleted file mode 100644
index d8aa826efe63..000000000000
--- a/metricbeat/module/traefik/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-TRAEFIK_HOST=traefik
-TRAEFIK_API_PORT=8080
diff --git a/metricbeat/module/traefik/test_traefik.py b/metricbeat/module/traefik/test_traefik.py
index de8de5f8b847..be42e4d676f0 100644
--- a/metricbeat/module/traefik/test_traefik.py
+++ b/metricbeat/module/traefik/test_traefik.py
@@ -25,5 +25,5 @@ def test_health(self, metricset):
         self.check_metricset("traefik", metricset, self.get_hosts(), self.FIELDS + ["service.name"])
 
     def get_hosts(self):
-        return [os.getenv('TRAEFIK_HOST', 'localhost') + ':' +
+        return [os.getenv('TRAEFIK_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('TRAEFIK_API_PORT', '8080')]
diff --git a/metricbeat/module/uwsgi/_meta/env b/metricbeat/module/uwsgi/_meta/env
deleted file mode 100644
index 2f9a17b57d49..000000000000
--- a/metricbeat/module/uwsgi/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-UWSGI_STAT_TCP_SERVER=tcp://uwsgi_tcp:9191
-UWSGI_STAT_HTTP_SERVER=http://uwsgi_http:9192
diff --git a/metricbeat/module/zookeeper/_meta/env b/metricbeat/module/zookeeper/_meta/env
deleted file mode 100644
index 6a063ecec28a..000000000000
--- a/metricbeat/module/zookeeper/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-ZOOKEEPER_HOST=zookeeper
-ZOOKEEPER_PORT=2181
diff --git a/metricbeat/tests/system/test_aerospike.py b/metricbeat/tests/system/test_aerospike.py
index 3f957e15885c..9d44794603a2 100644
--- a/metricbeat/tests/system/test_aerospike.py
+++ b/metricbeat/tests/system/test_aerospike.py
@@ -16,5 +16,5 @@ def test_namespace(self):
         self.check_metricset("aerospike", "namespace", self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('AEROSPIKE_HOST', 'localhost') + ':' +
+        return [os.getenv('AEROSPIKE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('AEROSPIKE_PORT', '3000')]
diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py
index fd1645181201..a74f7b980896 100644
--- a/metricbeat/tests/system/test_apache.py
+++ b/metricbeat/tests/system/test_apache.py
@@ -78,7 +78,7 @@ def verify_fields(self, evt):
         # There are more fields that could be checked.
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_HOST', 'localhost') + ':' +
+        return ['http://' + os.getenv('APACHE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('APACHE_PORT', '80')]
 
 
@@ -93,5 +93,5 @@ def verify_fields(self, evt):
             self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys())
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_OLD_HOST', 'localhost') + ':' +
+        return ['http://' + os.getenv('APACHE_OLD_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('APACHE_PORT', '80')]
diff --git a/metricbeat/tests/system/test_ceph.py b/metricbeat/tests/system/test_ceph.py
index 2e5d91c2e273..fb90acf8a283 100644
--- a/metricbeat/tests/system/test_ceph.py
+++ b/metricbeat/tests/system/test_ceph.py
@@ -24,5 +24,5 @@ def test_ceph(self, metricset):
         self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('CEPH_HOST', 'localhost') + ':' +
+        return [os.getenv('CEPH_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('CEPH_PORT', '5000')]
diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py
index 9a8ad67cc720..0804e5e0bee9 100644
--- a/metricbeat/tests/system/test_config.py
+++ b/metricbeat/tests/system/test_config.py
@@ -27,4 +27,4 @@ def test_invalid_config_with_removed_settings(self):
                                  " has been removed")
 
     def get_host(self):
-        return 'http://' + os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200')
+        return 'http://' + os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' + os.getenv('ES_PORT', '9200')
diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py
index f0d757803590..49d11b3f21ba 100644
--- a/metricbeat/tests/system/test_couchbase.py
+++ b/metricbeat/tests/system/test_couchbase.py
@@ -22,5 +22,5 @@ def test_couchbase(self, metricset):
         self.check_metricset("couchbase", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('COUCHBASE_HOST', 'localhost') + ':' +
+        return [os.getenv('COUCHBASE_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('COUCHBASE_PORT', '8091')]
diff --git a/metricbeat/tests/system/test_dropwizard.py b/metricbeat/tests/system/test_dropwizard.py
index a9ab7dbbe8de..651fb4968ec3 100644
--- a/metricbeat/tests/system/test_dropwizard.py
+++ b/metricbeat/tests/system/test_dropwizard.py
@@ -30,5 +30,5 @@ def test_dropwizard(self):
         self.assertTrue(len(output) >= 1)
 
     def get_hosts(self):
-        return [os.getenv('DROPWIZARD_HOST', 'localhost') + ':' +
+        return [os.getenv('DROPWIZARD_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('DROPWIZARD_PORT', '8080')]
diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py
index 40b7bce1a139..b30bcac1e208 100644
--- a/metricbeat/tests/system/test_envoyproxy.py
+++ b/metricbeat/tests/system/test_envoyproxy.py
@@ -31,5 +31,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ENVOYPROXY_HOST', 'localhost') + ':' +
+        return [os.getenv('ENVOYPROXY_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ENVOYPROXY_PORT', '9901')]
diff --git a/metricbeat/tests/system/test_golang.py b/metricbeat/tests/system/test_golang.py
index 19ddf8c9db4f..a053a2403fa0 100644
--- a/metricbeat/tests/system/test_golang.py
+++ b/metricbeat/tests/system/test_golang.py
@@ -43,5 +43,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('GOLANG_HOST', 'localhost') + ':' +
+        return ["http://" + os.getenv('GOLANG_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('GOLANG_PORT', '6060')]
diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py
index d25eadf6d567..64e6239be92f 100644
--- a/metricbeat/tests/system/test_http.py
+++ b/metricbeat/tests/system/test_http.py
@@ -76,4 +76,5 @@ def test_server(self):
         self.assert_fields_are_documented(evt)
 
     def get_host(self):
-        return "http://" + os.getenv('HTTP_HOST', 'localhost') + ':' + os.getenv('HTTP_PORT', '8080')
+        host = self.compose_hosts()[0]
+        return "http://" + os.getenv('HTTP_HOST', host) + ':' + os.getenv('HTTP_PORT', '8080')
diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py
index 921ee5aef254..3ca676999566 100644
--- a/metricbeat/tests/system/test_jolokia.py
+++ b/metricbeat/tests/system/test_jolokia.py
@@ -50,5 +50,5 @@ def test_jmx(self, mbean):
         assert evt["jolokia"]["test"]["gc"]["collection_count"] >= 0
 
     def get_hosts(self):
-        return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' +
+        return [os.getenv('JOLOKIA_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('JOLOKIA_PORT', '8778')]
diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index fbb7a5736d27..f3143819884f 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,7 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', 'localhost') + ':' +
+        return [os.getenv('KIBANA_HOST', self.compose_hosts()[1]) + ':' +
                 os.getenv('KIBANA_PORT', '5601')]
 
     def get_version(self):
diff --git a/metricbeat/tests/system/test_kubernetes.py b/metricbeat/tests/system/test_kubernetes.py
index 75f94dbd3317..1901774a2653 100644
--- a/metricbeat/tests/system/test_kubernetes.py
+++ b/metricbeat/tests/system/test_kubernetes.py
@@ -78,7 +78,7 @@ def _test_metricset(self, metricset, expected_events, hosts):
     def get_kubelet_hosts(cls):
         return [
             "http://" +
-            os.getenv('KUBELET_HOST', 'localhost') + ':' +
+            os.getenv('KUBELET_HOST', self.compose_hosts()[0]) + ':' +
             os.getenv('KUBELET_PORT', '10255')
         ]
 
@@ -86,6 +86,6 @@ def get_kubelet_hosts(cls):
     def get_kube_state_hosts(cls):
         return [
             "http://" +
-            os.getenv('KUBE_STATE_METRICS_HOST', 'localhost') + ':' +
+            os.getenv('KUBE_STATE_METRICS_HOST', self.compose_hosts()[1]) + ':' +
             os.getenv('KUBE_STATE_METRICS_PORT', '18080')
         ]
diff --git a/metricbeat/tests/system/test_logstash.py b/metricbeat/tests/system/test_logstash.py
index 8280bfb0223a..816aaf11cd7e 100644
--- a/metricbeat/tests/system/test_logstash.py
+++ b/metricbeat/tests/system/test_logstash.py
@@ -55,5 +55,5 @@ def test_node_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('LOGSTASH_HOST', 'localhost') + ':' +
+        return [os.getenv('LOGSTASH_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('LOGSTASH_PORT', '9600')]
diff --git a/metricbeat/tests/system/test_memcached.py b/metricbeat/tests/system/test_memcached.py
index ede980b0581c..c41d34b7cc66 100644
--- a/metricbeat/tests/system/test_memcached.py
+++ b/metricbeat/tests/system/test_memcached.py
@@ -31,5 +31,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MEMCACHED_HOST', 'localhost') + ':' +
+        return [os.getenv('MEMCACHED_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MEMCACHED_PORT', '11211')]
diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py
index 3a8493584a47..88327e80a23b 100644
--- a/metricbeat/tests/system/test_mongodb.py
+++ b/metricbeat/tests/system/test_mongodb.py
@@ -36,5 +36,5 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MONGODB_HOST', 'localhost') + ':' +
+        return [os.getenv('MONGODB_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MONGODB_PORT', '27017')]
diff --git a/metricbeat/tests/system/test_munin.py b/metricbeat/tests/system/test_munin.py
index e1618202fa49..391d8f845d5d 100644
--- a/metricbeat/tests/system/test_munin.py
+++ b/metricbeat/tests/system/test_munin.py
@@ -34,5 +34,5 @@ def test_munin_node(self):
         assert evt["munin"][namespace]["cpu"]["user"] > 0
 
     def get_hosts(self):
-        return [os.getenv('MUNIN_HOST', 'localhost') + ':' +
+        return [os.getenv('MUNIN_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('MUNIN_PORT', '4949')]
diff --git a/metricbeat/tests/system/test_mysql.py b/metricbeat/tests/system/test_mysql.py
index 987da6eb88a6..4087b141e49f 100644
--- a/metricbeat/tests/system/test_mysql.py
+++ b/metricbeat/tests/system/test_mysql.py
@@ -40,4 +40,6 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MYSQL_DSN', 'root:test@tcp(localhost:3306)/')]
+        host = os.getenv('MYSQL_HOST', self.compose_hosts()[0])
+        port = os.getenv('MYSQL_PORT', '3306')
+        return [os.getenv('MYSQL_DSN', 'root:test@tcp({}:{})/'.format(host, port))]
diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py
index edb5405a404d..4049f96748a1 100644
--- a/metricbeat/tests/system/test_phpfpm.py
+++ b/metricbeat/tests/system/test_phpfpm.py
@@ -34,5 +34,5 @@ def test_info(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('PHPFPM_HOST', 'localhost') + ':' +
+        return [os.getenv('PHPFPM_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('PHPFPM_PORT', '81')]
diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py
index 5150d9ab28b1..6f2e11937458 100644
--- a/metricbeat/tests/system/test_postgresql.py
+++ b/metricbeat/tests/system/test_postgresql.py
@@ -19,8 +19,15 @@ def common_checks(self, output):
             self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv("POSTGRESQL_DSN")], os.getenv("POSTGRESQL_USERNAME"), \
-            os.getenv("POSTGRESQL_PASSWORD")
+        username = "postgres"
+        host = self.compose_hosts()[0]
+        port = 5432
+        dsn = "postgres://{}:{}?sslmode=disable".format(host, port)
+        return (
+            [os.getenv("POSTGRESQL_DSN", dsn)],
+            os.getenv("POSTGRESQL_USERNAME", username),
+            os.getenv("POSTGRESQL_PASSWORD"),
+        )
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     @attr('integration')
diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py
index 535395460df8..ccf6bbf220fa 100644
--- a/metricbeat/tests/system/test_prometheus.py
+++ b/metricbeat/tests/system/test_prometheus.py
@@ -34,5 +34,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('PROMETHEUS_HOST', 'localhost') + ':' +
+        return ["http://" + os.getenv('PROMETHEUS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('PROMETHEUS_PORT', '9090')]
diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py
index 7871b2d17974..6d35f9d4d4cd 100644
--- a/metricbeat/tests/system/test_zookeeper.py
+++ b/metricbeat/tests/system/test_zookeeper.py
@@ -51,5 +51,5 @@ def test_output(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' +
+        return [os.getenv('ZOOKEEPER_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('ZOOKEEPER_PORT', '2181')]

From a5722798a16237d5d42e3496e05b5350390cbfd1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 08:56:17 +0200
Subject: [PATCH 15/88] Run redis python tests with two versions

---
 metricbeat/docker-compose.yml         | 6 ++++++
 metricbeat/tests/system/test_redis.py | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 30ac09530b77..8f6bf83d34ce 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -134,6 +134,12 @@ services:
       file: ./module/redis/docker-compose.yml
       service: redis
 
+  redis_3_2_12:
+    extends:
+      file: ./module/redis/docker-compose.yml
+      service: redis
+    image: redis:3.2.12-alpine
+
   traefik:
     build: ./module/traefik/_meta
 
diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 94b106ab929c..8fb96e4c8bd3 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -121,3 +121,7 @@ def test_module_processors(self):
     def get_hosts(self):
         return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]
+
+
+class Test_3_2_12(Test):
+    COMPOSE_SERVICES = ['redis_3_2_12']

From 8dd0a98df538d6f017b19bc09283dd1962d68f05 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 10:54:53 +0200
Subject: [PATCH 16/88] Add compose runner for kafka

---
 libbeat/tests/compose/wrapper.go              |   5 +-
 metricbeat/docker-compose.yml                 |  15 +-
 metricbeat/module/kafka/docker-compose.yml    |   8 +
 metricbeat/module/kafka/mtest/runner.go       |  48 ++++++
 .../partition/partition_integration_test.go   | 146 ++++++++----------
 .../redis/info/info_integration_test.go       |  11 +-
 .../keyspace/keyspace_integration_test.go     |   5 +-
 metricbeat/module/redis/mtest/runner.go       |  19 +++
 8 files changed, 159 insertions(+), 98 deletions(-)
 create mode 100644 metricbeat/module/kafka/docker-compose.yml
 create mode 100644 metricbeat/module/kafka/mtest/runner.go

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 2e67ef0f7851..a548a424de23 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,10 +20,8 @@ package compose
 import (
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -77,11 +75,14 @@ func (c *wrapperContainer) Host() string {
 	if len(ip) == 0 {
 		return ""
 	}
+	return ip
 
+	/* TODO:
 	for _, port := range c.info.Ports {
 		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
 	}
 	return ""
+	*/
 }
 
 func (d *wrapperDriver) LockFile() string {
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 6a7230835f01..1fcb07d29c37 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -69,20 +69,23 @@ services:
     build: ./module/jolokia/_meta
 
   kafka:
-    build:
-      context: ./module/kafka/_meta
-      args:
-        KAFKA_VERSION: 2.0.0
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
 
   kafka_1_1_0:
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
     build:
-      context: ./module/kafka/_meta
       args:
         KAFKA_VERSION: 1.1.0
 
   kafka_0_10_2:
+    extends:
+      file: ./module/kafka/docker-compose.yml
+      service: kafka
     build:
-      context: ./module/kafka/_meta
       args:
         KAFKA_VERSION: 0.10.2.1
 
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
new file mode 100644
index 000000000000..14041d028ea5
--- /dev/null
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -0,0 +1,8 @@
+version: "2.4"
+
+services:
+  kafka:
+    build:
+      context: ./_meta
+      args:
+        KAFKA_VERSION: 2.0.0
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
new file mode 100644
index 000000000000..5aba5e2e4bf8
--- /dev/null
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -0,0 +1,48 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service: "kafka",
+		Options: map[string][]string{
+			"KAFKA_VERSION": []string{
+				"0.10.2.1",
+				"1.1.0",
+				"2.0.0",
+			},
+		},
+		Parallel: true,
+	}
+
+	DataRunner = compose.TestRunner{
+		Service: "kafka",
+		Options: map[string][]string{
+			"KAFKA_VERSION": []string{
+				"1.1.0",
+			},
+		},
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index dc696c26b1d2..22b631c74ff8 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -22,7 +22,7 @@ package partition
 import (
 	"fmt"
 	"math/rand"
-	"os"
+	"net"
 	"strconv"
 	"testing"
 	"time"
@@ -34,91 +34,89 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-)
-
-const (
-	kafkaDefaultHost = "localhost"
-	kafkaDefaultPort = "9092"
+	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kafka")
+	t.Parallel()
 
-	generateKafkaData(t, "metricbeat-generate-data")
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		generateKafkaData(t, "metricbeat-generate-data", host)
 
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig(""))
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig("", host))
+		err := mbtest.WriteEventsReporterV2(ms, t, "")
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}})
 }
 
 func TestTopic(t *testing.T) {
-
-	compose.EnsureUp(t, "kafka")
+	t.Parallel()
 
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-	id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-	testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+	mtest.DataRunner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
+		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 
-	// Create initial topic
-	generateKafkaData(t, testTopic)
+		// Create initial topic
+		generateKafkaData(t, testTopic, host)
 
-	f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic))
-	dataBefore, err := mbtest.ReportingFetchV2(f)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-	if len(dataBefore) == 0 {
-		t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-	}
-	t.Logf("before: %v", dataBefore)
+		f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, host))
+		dataBefore, err := mbtest.ReportingFetchV2(f)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+		if len(dataBefore) == 0 {
+			t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+		}
+		t.Logf("before: %v", dataBefore)
 
-	var n int64 = 10
-	var i int64 = 0
-	// Create n messages
-	for ; i < n; i++ {
-		generateKafkaData(t, testTopic)
-	}
+		var n int64 = 10
+		// Create n messages
+		for i := int64(0); i < n; i++ {
+			generateKafkaData(t, testTopic, host)
+		}
 
-	dataAfter, err := mbtest.ReportingFetchV2(f)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-	if len(dataAfter) == 0 {
-		t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-	}
-	t.Logf("after: %v", dataAfter)
+		dataAfter, err := mbtest.ReportingFetchV2(f)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+		if len(dataAfter) == 0 {
+			t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+		}
+		t.Logf("after: %v", dataAfter)
 
-	// Checks that no new topics / partitions were added
-	assert.True(t, len(dataBefore) == len(dataAfter))
+		// Checks that no new topics / partitions were added
+		assert.True(t, len(dataBefore) == len(dataAfter))
 
-	var offsetBefore int64 = 0
-	var offsetAfter int64 = 0
+		var offsetBefore int64 = 0
+		var offsetAfter int64 = 0
 
-	// Its possible that other topics exists -> select the right data
-	for _, data := range dataBefore {
-		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-			offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		// Its possible that other topics exists -> select the right data
+		for _, data := range dataBefore {
+			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+				offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			}
 		}
-	}
 
-	for _, data := range dataAfter {
-		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-			offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		for _, data := range dataAfter {
+			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+				offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			}
 		}
-	}
 
-	// Compares offset before and after
-	if offsetBefore+n != offsetAfter {
-		t.Errorf("Offset before: %v", offsetBefore)
-		t.Errorf("Offset after: %v", offsetAfter)
-	}
-	assert.True(t, offsetBefore+n == offsetAfter)
+		// Compares offset before and after
+		if offsetBefore+n != offsetAfter {
+			t.Errorf("Offset before: %v", offsetBefore)
+			t.Errorf("Offset after: %v", offsetAfter)
+		}
+		assert.True(t, offsetBefore+n == offsetAfter)
+	}})
 }
 
-func generateKafkaData(t *testing.T, topic string) {
+func generateKafkaData(t *testing.T, topic string, host string) {
 	t.Logf("Send Kafka Event to topic: %v", topic)
 
 	config := sarama.NewConfig()
@@ -128,7 +126,7 @@ func generateKafkaData(t *testing.T, topic string) {
 	config.Producer.Retry.Backoff = 500 * time.Millisecond
 	config.Metadata.Retry.Max = 20
 	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{getTestKafkaHost()}, config)
+	client, err := sarama.NewClient([]string{net.JoinHostPort(host, "9092")}, config)
 	if err != nil {
 		t.Errorf("%s", err)
 		t.FailNow()
@@ -156,7 +154,7 @@ func generateKafkaData(t *testing.T, topic string) {
 	}
 }
 
-func getConfig(topic string) map[string]interface{} {
+func getConfig(topic string, host string) map[string]interface{} {
 	var topics []string
 	if topic != "" {
 		topics = []string{topic}
@@ -165,25 +163,7 @@ func getConfig(topic string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"partition"},
-		"hosts":      []string{getTestKafkaHost()},
+		"hosts":      []string{net.JoinHostPort(host, "9092")},
 		"topics":     topics,
 	}
 }
-
-func getTestKafkaHost() string {
-	return fmt.Sprintf("%v:%v",
-		getenv("KAFKA_HOST", kafkaDefaultHost),
-		getenv("KAFKA_PORT", kafkaDefaultPort),
-	)
-}
-
-func getenv(name, defaultValue string) string {
-	return strDefault(os.Getenv(name), defaultValue)
-}
-
-func strDefault(a, defaults string) string {
-	if len(a) == 0 {
-		return defaults
-	}
-	return a
-}
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 356aacc910aa..5152067911f8 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -20,6 +20,7 @@
 package info
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -105,7 +106,7 @@ func TestData(t *testing.T) {
 
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -117,7 +118,7 @@ func addPassword(host, pass string) error {
 
 // resetPassword changes the password to the redis DB.
 func resetPassword(host, currentPass string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -134,7 +135,7 @@ func resetPassword(host, currentPass string) error {
 
 // writeToRedis will write to the default DB 0.
 func writeToRedis(host string) error {
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		return err
 	}
@@ -144,11 +145,11 @@ func writeToRedis(host string) error {
 	return err
 }
 
-func getConfig(password, redisHost string) map[string]interface{} {
+func getConfig(password, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
-		"hosts":      []string{redisHost},
+		"hosts":      []string{net.JoinHostPort(host, "6379")},
 		"password":   password,
 	}
 }
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index be4d5cfc4176..14a38d7210ee 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -20,6 +20,7 @@
 package keyspace
 
 import (
+	"net"
 	"strings"
 	"testing"
 
@@ -79,7 +80,7 @@ func TestData(t *testing.T) {
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
-	c, err := rd.Dial("tcp", host)
+	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
 	if err != nil {
 		t.Fatal("connect", err)
 	}
@@ -94,6 +95,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},
-		"hosts":      []string{host},
+		"hosts":      []string{net.JoinHostPort(host, "6379")},
 	}
 }
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 343066c8ab0a..19001b8e4934 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -1,3 +1,22 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
 package mtest
 
 import (

From 38cc0819bb1bacd4717e082650e232aa1528833a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 11:18:38 +0200
Subject: [PATCH 17/88] Use docker compose 2.1 version

---
 metricbeat/docker-compose.yml              | 2 +-
 metricbeat/module/kafka/docker-compose.yml | 2 +-
 metricbeat/module/redis/docker-compose.yml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 1fcb07d29c37..721a8f933186 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   beat:
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index 14041d028ea5..9a6fdade8fb0 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   kafka:
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index f8e98e466851..9fac97dce56e 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -1,4 +1,4 @@
-version: "2.4"
+version: "2.1"
 
 services:
   redis:

From b3981f71426747032c671373c0c013639a0d8a79 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 21 Aug 2018 15:30:06 +0200
Subject: [PATCH 18/88] Use kafka runner in consumergroup metricset

---
 .../consumergroup_integration_test.go         | 65 ++++++-------------
 .../partition/partition_integration_test.go   |  2 +-
 2 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index df67af8da077..91bdeeb22e06 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -20,9 +20,8 @@
 package consumergroup
 
 import (
-	"fmt"
 	"io"
-	"os"
+	"net"
 	"testing"
 	"time"
 
@@ -31,61 +30,39 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-)
-
-const (
-	kafkaDefaultHost = "localhost"
-	kafkaDefaultPort = "9092"
+	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kafka")
-
-	c, err := startConsumer(t, "metricbeat-test")
-	if err != nil {
-		t.Fatal(errors.Wrap(err, "starting kafka consumer"))
-	}
-	defer c.Close()
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		c, err := startConsumer(t, "metricbeat-test", host)
+		if err != nil {
+			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
+		}
+		defer c.Close()
 
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
-	for retries := 0; retries < 3; retries++ {
-		err = mbtest.WriteEventsReporterV2(ms, t, "")
-		if err == nil {
-			return
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig(host))
+		for retries := 0; retries < 3; retries++ {
+			err = mbtest.WriteEventsReporterV2(ms, t, "")
+			if err == nil {
+				return
+			}
+			time.Sleep(500 * time.Millisecond)
 		}
-		time.Sleep(500 * time.Millisecond)
-	}
-	t.Fatal("write", err)
+		t.Fatal("write", err)
+	}})
 }
 
-func startConsumer(t *testing.T, topic string) (io.Closer, error) {
-	brokers := []string{getTestKafkaHost()}
+func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
+	brokers := []string{net.JoinHostPort(host, "9092")}
 	topics := []string{topic}
 	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"consumergroup"},
-		"hosts":      []string{getTestKafkaHost()},
-	}
-}
-
-func getTestKafkaHost() string {
-	return fmt.Sprintf("%v:%v",
-		getenv("KAFKA_HOST", kafkaDefaultHost),
-		getenv("KAFKA_PORT", kafkaDefaultPort),
-	)
-}
-
-func getenv(name, defaultValue string) string {
-	return strDefault(os.Getenv(name), defaultValue)
-}
-
-func strDefault(a, defaults string) string {
-	if len(a) == 0 {
-		return defaults
+		"hosts":      []string{net.JoinHostPort(host, "9092")},
 	}
-	return a
 }
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 22b631c74ff8..cb19fa048d4f 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -56,7 +56,7 @@ func TestTopic(t *testing.T) {
 
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-	mtest.DataRunner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
+	mtest.Runner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
 		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
 		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 

From b6735ea0f958f5eae835e922bda0e2e8f11b67b0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 12:27:23 +0200
Subject: [PATCH 19/88] Retry on docker compose up to avoid failures by system
 limits

---
 libbeat/tests/compose/wrapper.go | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index a548a424de23..84911d3a4292 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -23,6 +23,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -131,7 +132,19 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		args = append(args, service)
 	}
 
-	return d.cmd(ctx, "up", args...).Run()
+	for {
+		// It can fail if we have reached some system limit, specially
+		// number of networks, retry while the context is not done
+		err := d.cmd(ctx, "up", args...).Run()
+		if err == nil {
+			return nil
+		}
+		select {
+		case <-time.After(time.Second):
+		case <-ctx.Done():
+			return err
+		}
+	}
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {

From f9f0cd1078d754a8c85532b273d96f2b022ab0c0 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 19:06:29 +0200
Subject: [PATCH 20/88] Use compose runner for aerospike

---
 metricbeat/docker-compose.yml                 |  4 +-
 metricbeat/module/aerospike/_meta/Dockerfile  |  5 +-
 .../module/aerospike/docker-compose.yml       | 14 ++++++
 .../aerospike/{testing.go => mtest/runner.go} | 49 ++++++++++---------
 .../namespace/namespace_integration_test.go   | 21 ++++----
 5 files changed, 57 insertions(+), 36 deletions(-)
 create mode 100644 metricbeat/module/aerospike/docker-compose.yml
 rename metricbeat/module/aerospike/{testing.go => mtest/runner.go} (52%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 721a8f933186..3bef2cb90209 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -15,7 +15,9 @@ services:
 
   # Modules
   aerospike:
-    build: ./module/aerospike/_meta
+    extends:
+      file: ./module/aerospike/docker-compose.yml
+      service: aerospike
 
   apache:
     build: ./module/apache/_meta
diff --git a/metricbeat/module/aerospike/_meta/Dockerfile b/metricbeat/module/aerospike/_meta/Dockerfile
index 9693bbde4c27..e5d1c37bbd2c 100644
--- a/metricbeat/module/aerospike/_meta/Dockerfile
+++ b/metricbeat/module/aerospike/_meta/Dockerfile
@@ -1,4 +1,5 @@
-FROM aerospike:3.9.0
+ARG AEROSPIKE_VERSION=3.9.0
+
+FROM aerospike:$AEROSPIKE_VERSION
 
 RUN apt-get update && apt-get install -y netcat
-HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 3000
diff --git a/metricbeat/module/aerospike/docker-compose.yml b/metricbeat/module/aerospike/docker-compose.yml
new file mode 100644
index 000000000000..6ba1a786ef4d
--- /dev/null
+++ b/metricbeat/module/aerospike/docker-compose.yml
@@ -0,0 +1,14 @@
+version: '2.1'
+
+services:
+  aerospike:
+    build:
+      context: ./_meta
+      args:
+        AEROSPIKE_VERSION: ${AEROSPIKE_VERSION:-3.9.0}
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "nc -z localhost 3000"
+      interval: 1s
+      retries: 90
diff --git a/metricbeat/module/aerospike/testing.go b/metricbeat/module/aerospike/mtest/runner.go
similarity index 52%
rename from metricbeat/module/aerospike/testing.go
rename to metricbeat/module/aerospike/mtest/runner.go
index 298634e85c07..ddb50d85c539 100644
--- a/metricbeat/module/aerospike/testing.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -15,34 +15,35 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package aerospike
+// +build integration
+
+package mtest
 
 import (
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing used in the aerospike MetricSets.
-
-// GetAerospikeEnvHost returns the hostname of the Aerospike server to use for
-// testing. It reads the value from the AEROSPIKE_HOST environment variable and
-// returns localhost if it is not set.
-func GetAerospikeEnvHost() string {
-	host := os.Getenv("AEROSPIKE_HOST")
-
-	if len(host) == 0 {
-		host = "localhost"
+var (
+	Runner = compose.TestRunner{
+		Service: "aerospike",
+		Options: map[string][]string{
+			"AEROSPIKE_VERSION": []string{
+				"3.9.0",
+				"3.13.0.11",
+				"3.16.0.6",
+				"4.3.0.2",
+			},
+		},
+		Parallel: true,
 	}
-	return host
-}
 
-// GetAerospikeEnvPort returns the port of the Aerospike server to use for
-// testing. It reads the value from the AEROSPIKE_PORT environment variable and
-// returns 3000 if it is not set.
-func GetAerospikeEnvPort() string {
-	port := os.Getenv("AEROSPIKE_PORT")
-
-	if len(port) == 0 {
-		port = "3000"
+	DataRunner = compose.TestRunner{
+		Service: "aerospike",
+		Options: map[string][]string{
+			"AEROSPIKE_VERSION": []string{
+				"3.9.0",
+			},
+		},
+		Parallel: true,
 	}
-	return port
-}
+)
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 03fd7ed4397a..29f37afdb720 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -20,27 +20,30 @@
 package namespace
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/aerospike"
+	"github.com/elastic/beats/metricbeat/module/aerospike/mtest"
 )
 
 func TestData(t *testing.T) {
-	compose.EnsureUp(t, "aerospike")
+	t.Parallel()
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
+		f := mbtest.NewEventsFetcher(t, getConfig(host))
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "aerospike",
 		"metricsets": []string{"namespace"},
-		"hosts":      []string{aerospike.GetAerospikeEnvHost() + ":" + aerospike.GetAerospikeEnvPort()},
+		"hosts":      []string{net.JoinHostPort(host, "3000")},
 	}
 }

From a7382e6537bb2f95aca6bcbfc35358cb8ed69158 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:31:01 +0200
Subject: [PATCH 21/88] Inject a runner controller instead of just the host

---
 libbeat/tests/compose/runner.go               |  35 ++++-
 metricbeat/module/aerospike/mtest/runner.go   |  16 +--
 .../namespace/namespace_integration_test.go   |   6 +-
 .../consumergroup_integration_test.go         |   8 +-
 metricbeat/module/kafka/mtest/runner.go       |  10 --
 .../partition/partition_integration_test.go   | 132 +++++++++---------
 .../redis/info/info_integration_test.go       |  38 +++--
 .../keyspace/keyspace_integration_test.go     |  32 ++---
 metricbeat/module/redis/mtest/runner.go       |  13 --
 9 files changed, 133 insertions(+), 157 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 4b8c9912f1dc..977d9857eed3 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -40,7 +40,7 @@ type TestRunner struct {
 	Timeout  int
 }
 
-type Suite map[string]func(t *testing.T, host string)
+type Suite map[string]func(t *testing.T, r R)
 
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
@@ -71,6 +71,12 @@ func (r *TestRunner) scenarios() []map[string]string {
 	return scenarios
 }
 
+func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) { test(t, ctl) })
+	}
+}
+
 func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	env := strings.ToUpper(r.Service) + "_HOST"
 	host := os.Getenv(env)
@@ -80,9 +86,11 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 
 	t.Logf("Test host overriden by %s=%s", env, host)
 
-	for name, test := range tests {
-		t.Run(name, func(t *testing.T) { test(t, host) })
+	ctl := &runnerControl{
+		host: host,
+		t:    t,
 	}
+	r.runSuite(t, tests, ctl)
 	return true
 }
 
@@ -136,10 +144,27 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 				t.Fatal(errors.Wrapf(err, "getting host for %s/%s", r.Service, desc))
 			}
 
-			for name, test := range tests {
-				t.Run(name, func(t *testing.T) { test(t, host) })
+			ctl := &runnerControl{
+				host:     host,
+				t:        t,
+				scenario: s,
 			}
+			r.runSuite(t, tests, ctl)
 		})
 
 	}
 }
+
+type R interface {
+	Host() string
+}
+
+type runnerControl struct {
+	t        *testing.T
+	host     string
+	scenario map[string]string
+}
+
+func (r *runnerControl) Host() string {
+	return r.host
+}
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index ddb50d85c539..24af7a2ab7e6 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -29,19 +29,9 @@ var (
 		Options: map[string][]string{
 			"AEROSPIKE_VERSION": []string{
 				"3.9.0",
-				"3.13.0.11",
-				"3.16.0.6",
-				"4.3.0.2",
-			},
-		},
-		Parallel: true,
-	}
-
-	DataRunner = compose.TestRunner{
-		Service: "aerospike",
-		Options: map[string][]string{
-			"AEROSPIKE_VERSION": []string{
-				"3.9.0",
+				//"3.13.0.11",
+				//"3.16.0.6",
+				//"4.3.0.2",
 			},
 		},
 		Parallel: true,
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 29f37afdb720..9baaebacd8d3 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -28,11 +28,11 @@ import (
 	"github.com/elastic/beats/metricbeat/module/aerospike/mtest"
 )
 
-func TestData(t *testing.T) {
+func TestNamespace(t *testing.T) {
 	t.Parallel()
 
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
+	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
+		f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
 		err := mbtest.WriteEvents(f, t)
 		if err != nil {
 			t.Fatal("write", err)
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 91bdeeb22e06..4f40288ba51c 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -33,15 +33,15 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
-func TestData(t *testing.T) {
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		c, err := startConsumer(t, "metricbeat-test", host)
+func TestConsumerGroup(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
+		c, err := startConsumer(t, "metricbeat-test", r.Host())
 		if err != nil {
 			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
 		}
 		defer c.Close()
 
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig(host))
+		ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 		for retries := 0; retries < 3; retries++ {
 			err = mbtest.WriteEventsReporterV2(ms, t, "")
 			if err == nil {
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index 5aba5e2e4bf8..e4d93295a39b 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -35,14 +35,4 @@ var (
 		},
 		Parallel: true,
 	}
-
-	DataRunner = compose.TestRunner{
-		Service: "kafka",
-		Options: map[string][]string{
-			"KAFKA_VERSION": []string{
-				"1.1.0",
-			},
-		},
-		Parallel: true,
-	}
 )
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index cb19fa048d4f..ac51227bb7f6 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -37,83 +37,79 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kafka/mtest"
 )
 
-func TestData(t *testing.T) {
+func TestPartition(t *testing.T) {
 	t.Parallel()
 
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		generateKafkaData(t, "metricbeat-generate-data", host)
+	logp.TestingSetup(logp.WithSelectors("kafka"))
 
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig("", host))
-		err := mbtest.WriteEventsReporterV2(ms, t, "")
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			generateKafkaData(t, "metricbeat-generate-data", r.Host())
 
-func TestTopic(t *testing.T) {
-	t.Parallel()
+			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Topic": func(t *testing.T, r compose.R) {
+			id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+
+			// Create initial topic
+			generateKafkaData(t, testTopic, r.Host())
+
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
+			dataBefore, err := mbtest.ReportingFetchV2(f)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+			if len(dataBefore) == 0 {
+				t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+			}
+			t.Logf("before: %v", dataBefore)
 
-	logp.TestingSetup(logp.WithSelectors("kafka"))
+			var n int64 = 10
+			// Create n messages
+			for i := int64(0); i < n; i++ {
+				generateKafkaData(t, testTopic, r.Host())
+			}
+
+			dataAfter, err := mbtest.ReportingFetchV2(f)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+			if len(dataAfter) == 0 {
+				t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+			}
+			t.Logf("after: %v", dataAfter)
+
+			// Checks that no new topics / partitions were added
+			assert.True(t, len(dataBefore) == len(dataAfter))
+
+			var offsetBefore int64 = 0
+			var offsetAfter int64 = 0
+
+			// Its possible that other topics exists -> select the right data
+			for _, data := range dataBefore {
+				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+					offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+				}
+			}
 
-	mtest.Runner.Run(t, compose.Suite{"Topic": func(t *testing.T, host string) {
-		id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-		testTopic := fmt.Sprintf("test-metricbeat-%s", id)
-
-		// Create initial topic
-		generateKafkaData(t, testTopic, host)
-
-		f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, host))
-		dataBefore, err := mbtest.ReportingFetchV2(f)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-		if len(dataBefore) == 0 {
-			t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-		}
-		t.Logf("before: %v", dataBefore)
-
-		var n int64 = 10
-		// Create n messages
-		for i := int64(0); i < n; i++ {
-			generateKafkaData(t, testTopic, host)
-		}
-
-		dataAfter, err := mbtest.ReportingFetchV2(f)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-		if len(dataAfter) == 0 {
-			t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-		}
-		t.Logf("after: %v", dataAfter)
-
-		// Checks that no new topics / partitions were added
-		assert.True(t, len(dataBefore) == len(dataAfter))
-
-		var offsetBefore int64 = 0
-		var offsetAfter int64 = 0
-
-		// Its possible that other topics exists -> select the right data
-		for _, data := range dataBefore {
-			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-				offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			for _, data := range dataAfter {
+				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+					offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+				}
 			}
-		}
 
-		for _, data := range dataAfter {
-			if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-				offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+			// Compares offset before and after
+			if offsetBefore+n != offsetAfter {
+				t.Errorf("Offset before: %v", offsetBefore)
+				t.Errorf("Offset after: %v", offsetAfter)
 			}
-		}
-
-		// Compares offset before and after
-		if offsetBefore+n != offsetAfter {
-			t.Errorf("Offset before: %v", offsetBefore)
-			t.Errorf("Offset after: %v", offsetAfter)
-		}
-		assert.True(t, offsetBefore+n == offsetAfter)
-	}})
+			assert.True(t, offsetBefore+n == offsetAfter)
+		}})
 }
 
 func generateKafkaData(t *testing.T, topic string, host string) {
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 5152067911f8..1c4f946a7b49 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -40,8 +40,16 @@ func TestInfo(t *testing.T) {
 	t.Parallel()
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, host string) {
-			f := mbtest.NewEventFetcher(t, getConfig("", host))
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
 			event, err := f.Fetch()
 			if err != nil {
 				t.Fatal("fetch", err)
@@ -54,56 +62,42 @@ func TestInfo(t *testing.T) {
 			server := event["server"].(common.MapStr)
 			assert.Equal(t, "standalone", server["mode"])
 		},
-		"Passwords": func(t *testing.T, redisHost string) {
+		"Passwords": func(t *testing.T, r compose.R) {
 			// Add password and ensure it gets reset
 			defer func() {
-				err := resetPassword(redisHost, password)
+				err := resetPassword(r.Host(), password)
 				if err != nil {
 					t.Fatal("resetting password", err)
 				}
 			}()
 
-			err := addPassword(redisHost, password)
+			err := addPassword(r.Host(), password)
 			if err != nil {
 				t.Fatal("adding password", err)
 			}
 
 			// Test Fetch metrics with missing password
-			f := mbtest.NewEventFetcher(t, getConfig("", redisHost))
+			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
 			_, err = f.Fetch()
 			if assert.Error(t, err, "missing password") {
 				assert.Contains(t, err, "NOAUTH Authentication required.")
 			}
 
 			// Config redis and metricset with an invalid password
-			f = mbtest.NewEventFetcher(t, getConfig("blah", redisHost))
+			f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
 			_, err = f.Fetch()
 			if assert.Error(t, err, "invalid password") {
 				assert.Contains(t, err, "ERR invalid password")
 			}
 
 			// Config redis and metricset with a valid password
-			f = mbtest.NewEventFetcher(t, getConfig(password, redisHost))
+			f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
 			_, err = f.Fetch()
 			assert.NoError(t, err, "valid password")
 		},
 	})
 }
 
-func TestData(t *testing.T) {
-	t.Parallel()
-
-	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		f := mbtest.NewEventFetcher(t, getConfig("", host))
-
-		err := mbtest.WriteEvent(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
-
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
 	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 14a38d7210ee..1d47088eaa73 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -36,11 +36,21 @@ func TestKeyspace(t *testing.T) {
 	t.Parallel()
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, host string) {
-			addEntry(t, host)
+		"Data": func(t *testing.T, r compose.R) {
+			addEntry(t, r.Host())
+
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+		"Fetch": func(t *testing.T, r compose.R) {
+			addEntry(t, r.Host())
 
 			// Fetch data
-			f := mbtest.NewEventsFetcher(t, getConfig(host))
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
 			events, err := f.Fetch()
 			if err != nil {
 				t.Fatal("fetch", err)
@@ -61,22 +71,6 @@ func TestKeyspace(t *testing.T) {
 	})
 }
 
-func TestData(t *testing.T) {
-	t.Parallel()
-
-	// TODO: Fix EnsureUp for this kind of scenarios
-	mtest.DataRunner.Run(t, compose.Suite{"Data": func(t *testing.T, host string) {
-		addEntry(t, host)
-
-		f := mbtest.NewEventsFetcher(t, getConfig(host))
-
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
-}
-
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 19001b8e4934..49e0353365ca 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -39,17 +39,4 @@ var (
 		},
 		Parallel: true,
 	}
-
-	DataRunner = compose.TestRunner{
-		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
-				"4.0.11",
-			},
-			"IMAGE_OS": []string{
-				"alpine",
-			},
-		},
-		Parallel: true,
-	}
 )

From 760cdc3cc68f92eeca559f2007627489525051fa Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:40:14 +0200
Subject: [PATCH 22/88] Remove local images on down

---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 84911d3a4292..b5026448d35a 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v").Run()
+	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From 9b9f5be15eee917ba78bbcc2816d9597c512616c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 20:46:14 +0200
Subject: [PATCH 23/88] Add more versions to aerospike

---
 metricbeat/module/aerospike/mtest/runner.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index 24af7a2ab7e6..29a0a6a69e0b 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -29,9 +29,9 @@ var (
 		Options: map[string][]string{
 			"AEROSPIKE_VERSION": []string{
 				"3.9.0",
-				//"3.13.0.11",
-				//"3.16.0.6",
-				//"4.3.0.2",
+				"3.13.0.11",
+				"3.16.0.6",
+				"4.3.0.2",
 			},
 		},
 		Parallel: true,

From f45c67e3dc2c88430dbc7bb7a4c42dd652ad3aa3 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 21:19:34 +0200
Subject: [PATCH 24/88] Use compose runner for apache module

---
 metricbeat/docker-compose.yml                 | 12 +++--
 metricbeat/module/apache/_meta/Dockerfile     |  3 +-
 .../module/apache/_meta/Dockerfile.2.4.12     |  4 --
 metricbeat/module/apache/docker-compose.yml   |  8 +++
 .../apache/{testing.go => mtest/runner.go}    | 29 ++++++-----
 .../module/apache/status/_meta/data.json      | 39 +++++++-------
 .../apache/status/status_integration_test.go  | 51 ++++++++++---------
 7 files changed, 79 insertions(+), 67 deletions(-)
 delete mode 100644 metricbeat/module/apache/_meta/Dockerfile.2.4.12
 create mode 100644 metricbeat/module/apache/docker-compose.yml
 rename metricbeat/module/apache/{testing.go => mtest/runner.go} (67%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 3bef2cb90209..b656c6163ed1 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -20,12 +20,18 @@ services:
       service: aerospike
 
   apache:
-    build: ./module/apache/_meta
+    extends:
+      file: ./module/apache/docker-compose.yml
+      service: apache
 
   apache_2_4_12:
+    extends:
+      file: ./module/apache/docker-compose.yml
+      service: apache
     build:
-      context: ./module/apache/_meta
-      dockerfile: Dockerfile.2.4.12
+      args:
+        APACHE_VERSION: 2.4.12
+
 
   ceph:
     build: ./module/ceph/_meta
diff --git a/metricbeat/module/apache/_meta/Dockerfile b/metricbeat/module/apache/_meta/Dockerfile
index b6239b871f91..9b5b6ddcc36d 100644
--- a/metricbeat/module/apache/_meta/Dockerfile
+++ b/metricbeat/module/apache/_meta/Dockerfile
@@ -1,4 +1,5 @@
-FROM httpd:2.4.20
+ARG APACHE_VERSION
+FROM httpd:$APACHE_VERSION
 RUN apt-get update && apt-get install -y curl
 HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
 COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/apache/_meta/Dockerfile.2.4.12 b/metricbeat/module/apache/_meta/Dockerfile.2.4.12
deleted file mode 100644
index f35ea2d95c03..000000000000
--- a/metricbeat/module/apache/_meta/Dockerfile.2.4.12
+++ /dev/null
@@ -1,4 +0,0 @@
-FROM httpd:2.4.12
-RUN apt-get update && apt-get install -y curl
-HEALTHCHECK --interval=1s --retries=90 CMD curl -f http://localhost
-COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf
diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml
new file mode 100644
index 000000000000..c7338df16662
--- /dev/null
+++ b/metricbeat/module/apache/docker-compose.yml
@@ -0,0 +1,8 @@
+version: '2.1'
+
+services:
+  apache:
+    build:
+      context: ./_meta
+      args:
+        APACHE_VERSION: ${APACHE_VERSION:-2.4.20}
diff --git a/metricbeat/module/apache/testing.go b/metricbeat/module/apache/mtest/runner.go
similarity index 67%
rename from metricbeat/module/apache/testing.go
rename to metricbeat/module/apache/mtest/runner.go
index 4b2e0571a359..b069377ee912 100644
--- a/metricbeat/module/apache/testing.go
+++ b/metricbeat/module/apache/mtest/runner.go
@@ -15,22 +15,23 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package apache
+// +build integration
+
+package mtest
 
 import (
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing the Apache module's MetricSets.
-
-// GetApacheEnvHost returns the apache server hostname to use for testing. It
-// reads the value from the APACHE_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetApacheEnvHost() string {
-	host := os.Getenv("APACHE_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
+var (
+	Runner = compose.TestRunner{
+		Service: "apache",
+		Options: map[string][]string{
+			"APACHE_VERSION": []string{
+				"2.4.12",
+				"2.4.20",
+			},
+		},
+		Parallel: true,
 	}
-	return host
-}
+)
diff --git a/metricbeat/module/apache/status/_meta/data.json b/metricbeat/module/apache/status/_meta/data.json
index e8e3d45845f6..171b0c54f953 100644
--- a/metricbeat/module/apache/status/_meta/data.json
+++ b/metricbeat/module/apache/status/_meta/data.json
@@ -2,30 +2,29 @@
     "@timestamp": "2017-10-12T08:05:34.853Z",
     "apache": {
         "status": {
-            "bytes_per_request": 78.7692,
-            "bytes_per_sec": 4.94686,
+            "bytes_per_request": 0,
+            "bytes_per_sec": 0,
             "connections": {
                 "async": {
                     "closing": 0,
-                    "keep_alive": 1,
+                    "keep_alive": 0,
                     "writing": 0
                 },
-                "total": 1
+                "total": 0
             },
             "cpu": {
                 "children_system": 0,
                 "children_user": 0,
-                "load": 3.99758,
-                "system": 16.39,
-                "user": 0.16
+                "system": 0,
+                "user": 0
             },
-            "hostname": "apache",
+            "hostname": "172.26.0.2:80",
             "load": {
-                "1": 170.11,
-                "15": 77.62,
-                "5": 167.51
+                "1": 1.3,
+                "15": 1.56,
+                "5": 1.34
             },
-            "requests_per_sec": 0.0628019,
+            "requests_per_sec": 0.666667,
             "scoreboard": {
                 "closing_connection": 0,
                 "dns_lookup": 0,
@@ -33,22 +32,22 @@
                 "idle_cleanup": 0,
                 "keepalive": 0,
                 "logging": 0,
-                "open_slot": 300,
+                "open_slot": 325,
                 "reading_request": 0,
                 "sending_reply": 1,
                 "starting_up": 0,
                 "total": 400,
-                "waiting_for_connection": 99
+                "waiting_for_connection": 74
             },
-            "total_accesses": 26,
-            "total_kbytes": 2,
+            "total_accesses": 2,
+            "total_kbytes": 0,
             "uptime": {
-                "server_uptime": 414,
-                "uptime": 414
+                "server_uptime": 3,
+                "uptime": 3
             },
             "workers": {
                 "busy": 1,
-                "idle": 99
+                "idle": 74
             }
         }
     },
@@ -57,7 +56,7 @@
         "name": "host.example.com"
     },
     "metricset": {
-        "host": "apache",
+        "host": "172.26.0.2:80",
         "module": "apache",
         "name": "status",
         "rtt": 115
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 01e7f70b53ad..986ecb0c19bf 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -20,47 +20,48 @@
 package status
 
 import (
+	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/apache"
+	"github.com/elastic/beats/metricbeat/module/apache/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "apache")
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check number of fields.
-	if len(event) < 11 {
-		t.Fatal("Too few top-level elements in the event")
-	}
-}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "apache")
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			// Check number of fields.
+			if len(event) < 11 {
+				t.Fatal("Too few top-level elements in the event")
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "apache",
 		"metricsets": []string{"status"},
-		"hosts":      []string{apache.GetApacheEnvHost()},
+		"hosts":      []string{net.JoinHostPort(host, "80")},
 	}
 }

From 821f71a39a1d48911b7b6e55f4c055b92ace5819 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 23 Aug 2018 21:28:50 +0200
Subject: [PATCH 25/88] Revert "Remove local images on down"

This reverts commit 760cdc3cc68f92eeca559f2007627489525051fa.
---
 libbeat/tests/compose/wrapper.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index b5026448d35a..84911d3a4292 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
+	return d.cmd(ctx, "down", "-v").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From d1321cca894ab29e6250a16ded3eef02a02391d3 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 24 Aug 2018 17:04:08 +0200
Subject: [PATCH 26/88] Revert "Revert "Remove local images on down""

This reverts commit 821f71a39a1d48911b7b6e55f4c055b92ace5819.
---
 libbeat/tests/compose/wrapper.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 84911d3a4292..6fddd46bfbf8 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -87,7 +87,7 @@ func (c *wrapperContainer) Host() string {
 }
 
 func (d *wrapperDriver) LockFile() string {
-	return d.Files[0] + "." + d.Name + ".lock"
+	return d.Files[0] + ".lock"
 }
 
 func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string) *exec.Cmd {
@@ -148,7 +148,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 }
 
 func (d *wrapperDriver) Down(ctx context.Context) error {
-	return d.cmd(ctx, "down", "-v").Run()
+	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
 
 func (d *wrapperDriver) Kill(ctx context.Context, signal string, service string) error {

From 13252a6818546f004c96a2ea7255f4d923fc9f9e Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 6 Sep 2018 18:10:48 +0200
Subject: [PATCH 27/88] Using public ports as host addresses

---
 libbeat/tests/compose/runner.go               | 11 ++++++++++
 libbeat/tests/compose/wrapper.go              | 20 ++++++++++++++-----
 .../module/aerospike/docker-compose.yml       |  2 ++
 .../namespace/namespace_integration_test.go   |  3 +--
 metricbeat/module/apache/docker-compose.yml   |  2 ++
 .../apache/status/status_integration_test.go  |  3 +--
 metricbeat/module/kafka/_meta/run.sh          |  4 ++--
 metricbeat/module/kafka/broker.go             | 11 +++++++---
 .../consumergroup_integration_test.go         |  5 ++---
 metricbeat/module/kafka/docker-compose.yml    |  2 ++
 .../partition/partition_integration_test.go   |  5 ++---
 metricbeat/module/redis/docker-compose.yml    |  2 ++
 .../redis/info/info_integration_test.go       |  9 ++++-----
 .../keyspace/keyspace_integration_test.go     |  5 ++---
 14 files changed, 56 insertions(+), 28 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 977d9857eed3..1646b5df2670 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -20,6 +20,7 @@ package compose
 import (
 	"fmt"
 	"math/rand"
+	"net"
 	"os"
 	"sort"
 	"strings"
@@ -168,3 +169,13 @@ type runnerControl struct {
 func (r *runnerControl) Host() string {
 	return r.host
 }
+
+func (r *runnerControl) Hostname() string {
+	hostname, _, _ := net.SplitHostPort(r.host)
+	return hostname
+}
+
+func (r *runnerControl) Port() string {
+	_, port, _ := net.SplitHostPort(r.host)
+	return port
+}
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 6fddd46bfbf8..2cac67a0ee51 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -20,8 +20,10 @@ package compose
 import (
 	"context"
 	"fmt"
+	"net"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 
@@ -64,8 +66,7 @@ func (c *wrapperContainer) Old() bool {
 	return strings.Contains(c.info.Status, "minute")
 }
 
-func (c *wrapperContainer) Host() string {
-	// TODO: Support multiple networks/ports
+func (c *wrapperContainer) privateHost() string {
 	var ip string
 	for _, net := range c.info.NetworkSettings.Networks {
 		if len(net.IPAddress) > 0 {
@@ -76,14 +77,23 @@ func (c *wrapperContainer) Host() string {
 	if len(ip) == 0 {
 		return ""
 	}
-	return ip
 
-	/* TODO:
 	for _, port := range c.info.Ports {
 		return net.JoinHostPort(ip, strconv.Itoa(int(port.PrivatePort)))
 	}
 	return ""
-	*/
+}
+
+func (c *wrapperContainer) publicHost() string {
+	for _, port := range c.info.Ports {
+		return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+	}
+	return ""
+}
+
+func (c *wrapperContainer) Host() string {
+	// TODO: Support multiple networks/ports
+	return c.publicHost()
 }
 
 func (d *wrapperDriver) LockFile() string {
diff --git a/metricbeat/module/aerospike/docker-compose.yml b/metricbeat/module/aerospike/docker-compose.yml
index 6ba1a786ef4d..ace5433662b7 100644
--- a/metricbeat/module/aerospike/docker-compose.yml
+++ b/metricbeat/module/aerospike/docker-compose.yml
@@ -6,6 +6,8 @@ services:
       context: ./_meta
       args:
         AEROSPIKE_VERSION: ${AEROSPIKE_VERSION:-3.9.0}
+    ports:
+      - "3000"
     healthcheck:
       test:
       - "CMD-SHELL"
diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 9baaebacd8d3..9643548c699c 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -20,7 +20,6 @@
 package namespace
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -44,6 +43,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "aerospike",
 		"metricsets": []string{"namespace"},
-		"hosts":      []string{net.JoinHostPort(host, "3000")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/apache/docker-compose.yml b/metricbeat/module/apache/docker-compose.yml
index c7338df16662..93433d34491e 100644
--- a/metricbeat/module/apache/docker-compose.yml
+++ b/metricbeat/module/apache/docker-compose.yml
@@ -6,3 +6,5 @@ services:
       context: ./_meta
       args:
         APACHE_VERSION: ${APACHE_VERSION:-2.4.20}
+    ports:
+    - 80
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 986ecb0c19bf..631bd0cc0b15 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -20,7 +20,6 @@
 package status
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -62,6 +61,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "apache",
 		"metricsets": []string{"status"},
-		"hosts":      []string{net.JoinHostPort(host, "80")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 36b42a6a0d5d..451e183c912b 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -21,8 +21,8 @@ wait_for_port 2181
 echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
-    --override delete.topic.enable=true --override advertised.host.name=${KAFKA_ADVERTISED_HOST} \
-    --override listeners=PLAINTEXT://0.0.0.0:9092 \
+    --override delete.topic.enable=true \
+    --override listeners=PLAINTEXT://:9092 \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092
diff --git a/metricbeat/module/kafka/broker.go b/metricbeat/module/kafka/broker.go
index dd5a15b47a88..2d241c14e880 100644
--- a/metricbeat/module/kafka/broker.go
+++ b/metricbeat/module/kafka/broker.go
@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/Shopify/sarama"
+	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/libbeat/common"
 )
@@ -102,7 +103,11 @@ func (b *Broker) Close() error {
 // Connect connects the broker to the configured host
 func (b *Broker) Connect() error {
 	if err := b.broker.Open(b.cfg); err != nil {
-		return err
+		return errors.Wrap(err, "opening broker connection")
+	}
+
+	if ok, err := b.broker.Connected(); !ok {
+		return errors.Wrap(err, "broker not connected")
 	}
 
 	if b.id != noID || !b.matchID {
@@ -113,13 +118,13 @@ func (b *Broker) Connect() error {
 	meta, err := queryMetadataWithRetry(b.broker, b.cfg, nil)
 	if err != nil {
 		closeBroker(b.broker)
-		return err
+		return errors.Wrap(err, "querying metadata")
 	}
 
 	other := findMatchingBroker(brokerAddress(b.broker), meta.Brokers)
 	if other == nil { // no broker found
 		closeBroker(b.broker)
-		return fmt.Errorf("No advertised broker with address %v found", b.Addr())
+		return fmt.Errorf("no advertised broker with address %v found", b.Addr())
 	}
 
 	debugf("found matching broker %v with id %v", other.Addr(), other.ID())
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 4f40288ba51c..8a6541057c7e 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -21,7 +21,6 @@ package consumergroup
 
 import (
 	"io"
-	"net"
 	"testing"
 	"time"
 
@@ -54,7 +53,7 @@ func TestConsumerGroup(t *testing.T) {
 }
 
 func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
-	brokers := []string{net.JoinHostPort(host, "9092")}
+	brokers := []string{host}
 	topics := []string{topic}
 	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
 }
@@ -63,6 +62,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"consumergroup"},
-		"hosts":      []string{net.JoinHostPort(host, "9092")},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index 9a6fdade8fb0..abd3e8f0aeda 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -6,3 +6,5 @@ services:
       context: ./_meta
       args:
         KAFKA_VERSION: 2.0.0
+    ports:
+    - 9092
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index ac51227bb7f6..3b0c8c507fad 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -22,7 +22,6 @@ package partition
 import (
 	"fmt"
 	"math/rand"
-	"net"
 	"strconv"
 	"testing"
 	"time"
@@ -122,7 +121,7 @@ func generateKafkaData(t *testing.T, topic string, host string) {
 	config.Producer.Retry.Backoff = 500 * time.Millisecond
 	config.Metadata.Retry.Max = 20
 	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{net.JoinHostPort(host, "9092")}, config)
+	client, err := sarama.NewClient([]string{host}, config)
 	if err != nil {
 		t.Errorf("%s", err)
 		t.FailNow()
@@ -159,7 +158,7 @@ func getConfig(topic string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
 		"metricsets": []string{"partition"},
-		"hosts":      []string{net.JoinHostPort(host, "9092")},
+		"hosts":      []string{host},
 		"topics":     topics,
 	}
 }
diff --git a/metricbeat/module/redis/docker-compose.yml b/metricbeat/module/redis/docker-compose.yml
index 9fac97dce56e..f0821dde978f 100644
--- a/metricbeat/module/redis/docker-compose.yml
+++ b/metricbeat/module/redis/docker-compose.yml
@@ -3,6 +3,8 @@ version: "2.1"
 services:
   redis:
     image: redis:${REDIS_VERSION:-4.0.11}-${IMAGE_OS:-alpine}
+    ports:
+      - 6379
     healthcheck:
       test:
       - "CMD-SHELL"
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 1c4f946a7b49..7d17df5b380a 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -20,7 +20,6 @@
 package info
 
 import (
-	"net"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -100,7 +99,7 @@ func TestInfo(t *testing.T) {
 
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -112,7 +111,7 @@ func addPassword(host, pass string) error {
 
 // resetPassword changes the password to the redis DB.
 func resetPassword(host, currentPass string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -129,7 +128,7 @@ func resetPassword(host, currentPass string) error {
 
 // writeToRedis will write to the default DB 0.
 func writeToRedis(host string) error {
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		return err
 	}
@@ -143,7 +142,7 @@ func getConfig(password, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"info"},
-		"hosts":      []string{net.JoinHostPort(host, "6379")},
+		"hosts":      []string{host},
 		"password":   password,
 	}
 }
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 1d47088eaa73..4109d3845684 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -20,7 +20,6 @@
 package keyspace
 
 import (
-	"net"
 	"strings"
 	"testing"
 
@@ -74,7 +73,7 @@ func TestKeyspace(t *testing.T) {
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
-	c, err := rd.Dial("tcp", net.JoinHostPort(host, "6379"))
+	c, err := rd.Dial("tcp", host)
 	if err != nil {
 		t.Fatal("connect", err)
 	}
@@ -89,6 +88,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "redis",
 		"metricsets": []string{"keyspace"},
-		"hosts":      []string{net.JoinHostPort(host, "6379")},
+		"hosts":      []string{host},
 	}
 }

From 01174617b8e414512e0e497e3f0e92586554eb24 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 7 Sep 2018 18:33:06 +0200
Subject: [PATCH 28/88] Inject context information into the compose containers

Some services need to know the addresses clients are going to use to
connect with them, for example Kafka brokers announce the address that
has to be used to connect with them. This information is not available
in the containers. This change injects some of this information so
services can configure themselves.
---
 libbeat/tests/compose/runner.go      |  4 +-
 libbeat/tests/compose/wrapper.go     | 68 +++++++++++++++++++++++++++-
 metricbeat/module/kafka/_meta/run.sh | 23 +++++++++-
 3 files changed, 90 insertions(+), 5 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 1646b5df2670..58d92255df2f 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -130,10 +130,12 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			err := project.Start(r.Service)
+			// Down() is "idempotent", Start() has several points where it can fail,
+			// so run Down() even if Start() fails.
+			defer project.Down()
 			if err != nil {
 				t.Fatal(err)
 			}
-			defer project.Down()
 
 			err = project.Wait(timeout, r.Service)
 			if err != nil {
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 2cac67a0ee51..5e4e3f60269a 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -18,11 +18,14 @@
 package compose
 
 import (
+	"archive/tar"
+	"bytes"
 	"context"
 	"fmt"
 	"net"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -86,7 +89,9 @@ func (c *wrapperContainer) privateHost() string {
 
 func (c *wrapperContainer) publicHost() string {
 	for _, port := range c.info.Ports {
-		return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+		if port.PublicPort != 0 {
+			return net.JoinHostPort("localhost", strconv.Itoa(int(port.PublicPort)))
+		}
 	}
 	return ""
 }
@@ -147,7 +152,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		// number of networks, retry while the context is not done
 		err := d.cmd(ctx, "up", args...).Run()
 		if err == nil {
-			return nil
+			return d.setupAdvertisedHost(ctx, service)
 		}
 		select {
 		case <-time.After(time.Second):
@@ -157,6 +162,64 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 	}
 }
 
+func writeToContainer(ctx context.Context, id, filename, content string) error {
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	now := time.Now()
+	err := tw.WriteHeader(&tar.Header{
+		Typeflag:   tar.TypeReg,
+		Name:       filepath.Base(filename),
+		Mode:       0100644,
+		Size:       int64(len(content)),
+		ModTime:    now,
+		AccessTime: now,
+		ChangeTime: now,
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to write tar header")
+	}
+	if _, err := tw.Write([]byte(content)); err != nil {
+		return errors.Wrap(err, "failed to write tar file")
+	}
+	if err := tw.Close(); err != nil {
+		return errors.Wrap(err, "failed to close tar")
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		return errors.Wrap(err, "failed to start docker client")
+	}
+	defer cli.Close()
+
+	opts := types.CopyToContainerOptions{}
+	err = cli.CopyToContainer(ctx, id, filepath.Dir(filename), bytes.NewReader(buf.Bytes()), opts)
+	if err != nil {
+		return errors.Wrapf(err, "failed to copy environment to container %s", id)
+	}
+	return nil
+}
+
+func (d *wrapperDriver) setupAdvertisedHost(ctx context.Context, service string) error {
+	containers, err := d.containers(ctx, Filter{State: AnyState}, service)
+	if err != nil {
+		return errors.Wrap(err, "setupAdvertisedHost")
+	}
+	if len(containers) == 0 {
+		return errors.Errorf("no containers for service %s", service)
+	}
+
+	for _, c := range containers {
+		w := &wrapperContainer{info: c}
+		content := fmt.Sprintf("SERVICE_HOST=%s", w.Host())
+
+		err := writeToContainer(ctx, c.ID, "/run/compose_env", content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (d *wrapperDriver) Down(ctx context.Context) error {
 	return d.cmd(ctx, "down", "-v", "--rmi=local").Run()
 }
@@ -206,6 +269,7 @@ func (d *wrapperDriver) containers(ctx context.Context, projectFilter Filter, fi
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to start docker client")
 	}
+	defer cli.Close()
 
 	var serviceFilters []filters.Args
 	if len(filter) == 0 {
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 451e183c912b..696b9507b4bc 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -1,6 +1,22 @@
 #!/bin/bash
 
-KAFKA_ADVERTISED_HOST=$(dig +short $HOSTNAME)
+# Check if KAFKA_ADVERTISED_HOST is set
+# if not wait to read it from file
+if [ -z "$KAFKA_ADVERTISED_HOST" ]; then
+	echo "KAFKA_ADVERTISED_HOST needed, will wait for it on /var/run/compose_env"
+	while true; do
+		if [ -f /run/compose_env ]; then
+			source /run/compose_env
+			KAFKA_ADVERTISED_HOST=$SERVICE_HOST
+		fi
+		if [ -n "$KAFKA_ADVERTISED_HOST" ]; then
+			# Remove it so it is not reused
+			> /run/compose_env
+			break
+		fi
+		sleep 1
+	done
+fi
 
 wait_for_port() {
     count=20
@@ -22,7 +38,10 @@ echo "Starting Kafka broker"
 mkdir -p ${KAFKA_LOGS_DIR}
 ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
     --override delete.topic.enable=true \
-    --override listeners=PLAINTEXT://:9092 \
+    --override listeners=INSIDE://localhost:9091,OUTSIDE://0.0.0.0:9092 \
+    --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \
+    --override listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT \
+    --override inter.broker.listener.name=INSIDE \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092

From 2ca7e84abe0bf488a10c8616496875429ba6d9b6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 7 Sep 2018 20:42:46 +0200
Subject: [PATCH 29/88] Keep environment when calling docker compose

---
 libbeat/tests/compose/wrapper.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 5e4e3f60269a..119b68280c39 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -117,7 +117,7 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if len(d.Environment) > 0 {
-		cmd.Env = d.Environment
+		cmd.Env = append(d.Environment, os.Environ()...)
 	}
 	return cmd
 }
@@ -154,6 +154,7 @@ func (d *wrapperDriver) Up(ctx context.Context, opts UpOptions, service string)
 		if err == nil {
 			return d.setupAdvertisedHost(ctx, service)
 		}
+
 		select {
 		case <-time.After(time.Second):
 		case <-ctx.Done():

From 2246ccbe614d4cf9a6c7bf4b0c804c26f4e642cf Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 12:33:29 +0200
Subject: [PATCH 30/88] pep8

---
 metricbeat/tests/system/test_redis.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index 188948238ff3..a1404390860a 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -123,8 +123,10 @@ def get_hosts(self):
         return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('REDIS_PORT', '6379')]
 
+
 class TestRedis4(Test):
     COMPOSE_SERVICES = ['redis_4']
 
+
 class TestRedis5(Test):
     COMPOSE_SERVICES = ['redis_5']

From 5e4335c3574596ba6f3fce1e501dc50631aadc23 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 13:49:25 +0200
Subject: [PATCH 31/88] Add type to simplify config of test runners

---
 libbeat/tests/compose/runner.go             | 12 ++++++++++--
 metricbeat/module/aerospike/mtest/runner.go |  4 ++--
 metricbeat/module/kafka/mtest/runner.go     |  4 ++--
 metricbeat/module/redis/mtest/runner.go     |  6 +++---
 4 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 58d92255df2f..4484e9b616e6 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -36,13 +36,15 @@ func init() {
 
 type TestRunner struct {
 	Service  string
-	Options  map[string][]string
+	Options  RunnerOptions
 	Parallel bool
 	Timeout  int
 }
 
 type Suite map[string]func(t *testing.T, r R)
 
+type RunnerOptions map[string][]string
+
 func (r *TestRunner) scenarios() []map[string]string {
 	n := 1
 	options := make(map[string][]string)
@@ -96,6 +98,8 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 }
 
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
+	t.Helper()
+
 	if r.runHostOverride(t, tests) {
 		return
 	}
@@ -105,7 +109,11 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		timeout = 300
 	}
 
-	for _, s := range r.scenarios() {
+	scenarios := r.scenarios()
+	if len(scenarios) == 0 {
+		t.Fatal("Test runner configuration doesn't produce scenarios")
+	}
+	for _, s := range scenarios {
 		var vars []string
 		for k, v := range s {
 			os.Setenv(k, v)
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index 29a0a6a69e0b..fbd3ef5b56c6 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -26,8 +26,8 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "aerospike",
-		Options: map[string][]string{
-			"AEROSPIKE_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"AEROSPIKE_VERSION": {
 				"3.9.0",
 				"3.13.0.11",
 				"3.16.0.6",
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index e4d93295a39b..e3b6a5c2e807 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -26,8 +26,8 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "kafka",
-		Options: map[string][]string{
-			"KAFKA_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"KAFKA_VERSION": {
 				"0.10.2.1",
 				"1.1.0",
 				"2.0.0",
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 49e0353365ca..6d3dbd546e83 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -26,13 +26,13 @@ import (
 var (
 	Runner = compose.TestRunner{
 		Service: "redis",
-		Options: map[string][]string{
-			"REDIS_VERSION": []string{
+		Options: compose.RunnerOptions{
+			"REDIS_VERSION": {
 				"3.2.12",
 				"4.0.11",
 				"5.0-rc",
 			},
-			"IMAGE_OS": []string{
+			"IMAGE_OS": {
 				"alpine",
 				"stretch",
 			},

From 9a21006f0487b9f428d393bdc8acb50983df3438 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 13:49:50 +0200
Subject: [PATCH 32/88] Run dropwizard tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../collector/collector_integration_test.go   | 114 ++++++++----------
 2 files changed, 51 insertions(+), 65 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index aa4d671695be..417518296b1f 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -41,6 +41,8 @@ services:
 
   dropwizard:
     build: ./module/dropwizard/_meta
+    ports:
+      - 8080
 
   elasticsearch:
     build: ./module/elasticsearch/_meta
diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go
index c4b3cf490ab5..8f4af491b007 100644
--- a/metricbeat/module/dropwizard/collector/collector_integration_test.go
+++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go
@@ -20,7 +20,6 @@
 package collector
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -30,83 +29,68 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "dropwizard")
+func TestCollector(t *testing.T) {
+	t.Parallel()
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
+	runner := compose.TestRunner{Service: "dropwizard"}
 
-	hasTag := false
-	doesntHaveTag := false
-	for _, event := range events {
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			events, err := f.Fetch()
 
-		ok, _ := event.HasKey("my_histogram")
-		if ok {
-			_, err := event.GetValue("tags")
-			if err == nil {
-				t.Fatal("write", "my_counter not supposed to have tags")
+			hasTag := false
+			doesntHaveTag := false
+			for _, event := range events {
+
+				ok, _ := event.HasKey("my_histogram")
+				if ok {
+					_, err := event.GetValue("tags")
+					if err == nil {
+						t.Fatal("write", "my_counter not supposed to have tags")
+					}
+					doesntHaveTag = true
+				}
+
+				ok, _ = event.HasKey("my_counter")
+				if ok {
+					tagsRaw, err := event.GetValue("tags")
+					if err != nil {
+						t.Fatal("write", err)
+					} else {
+						tags, ok := tagsRaw.(common.MapStr)
+						if !ok {
+							t.Fatal("write", "unable to cast tags to common.MapStr")
+						} else {
+							assert.Equal(t, len(tags), 1)
+							hasTag = true
+						}
+					}
+				}
+			}
+			assert.Equal(t, hasTag, true)
+			assert.Equal(t, doesntHaveTag, true)
+			if !assert.NoError(t, err) {
+				t.FailNow()
 			}
-			doesntHaveTag = true
-		}
 
-		ok, _ = event.HasKey("my_counter")
-		if ok {
-			tagsRaw, err := event.GetValue("tags")
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvents(f, t)
 			if err != nil {
 				t.Fatal("write", err)
-			} else {
-				tags, ok := tagsRaw.(common.MapStr)
-				if !ok {
-					t.Fatal("write", "unable to cast tags to common.MapStr")
-				} else {
-					assert.Equal(t, len(tags), 1)
-					hasTag = true
-				}
 			}
-		}
-	}
-	assert.Equal(t, hasTag, true)
-	assert.Equal(t, doesntHaveTag, true)
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "dropwizard")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getEnvHost() string {
-	host := os.Getenv("DROPWIZARD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("DROPWIZARD_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":       "dropwizard",
 		"metricsets":   []string{"collector"},
-		"hosts":        []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":        []string{host},
 		"namespace":    "testnamespace",
 		"metrics_path": "/test/metrics",
 		"enabled":      true,

From edc7982e5a49cf010ad978b9ebd3fc41c80b1fba Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 14:36:43 +0200
Subject: [PATCH 33/88] Run ES tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../elasticsearch_integration_test.go         | 103 +++++++-----------
 2 files changed, 43 insertions(+), 62 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 417518296b1f..180299e81a71 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -52,6 +52,8 @@ services:
       - "transport.host=127.0.0.1"
       - "http.host=0.0.0.0"
       - "xpack.security.enabled=false"
+    ports:
+      - 9200
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 5ceff6d7187f..2345cd5f07f1 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -22,9 +22,7 @@ package elasticsearch_test
 import (
 	"fmt"
 	"io/ioutil"
-	"net"
 	"net/http"
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -55,74 +53,55 @@ var metricSets = []string{
 	"shard",
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch")
-
-	host := net.JoinHostPort(getEnvHost(), getEnvPort())
-	err := createIndex(host)
-	assert.NoError(t, err)
-
-	err = enableTrialLicense(host)
-	assert.NoError(t, err)
-
-	err = createMLJob(host)
-	assert.NoError(t, err)
-
-	for _, metricSet := range metricSets {
-		t.Run(metricSet, func(t *testing.T) {
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet))
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
+func TestElasticsearch(t *testing.T) {
+	runner := compose.TestRunner{Service: "elasticsearch"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			host := r.Host()
+			err := createIndex(host)
+			assert.NoError(t, err)
+
+			err = enableTrialLicense(host)
+			assert.NoError(t, err)
+
+			err = createMLJob(host)
+			assert.NoError(t, err)
+
+			for _, metricSet := range metricSets {
+				t.Run(metricSet, func(t *testing.T) {
+					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
+					events, errs := mbtest.ReportingFetchV2(f)
+
+					assert.Empty(t, errs)
+					if !assert.NotEmpty(t, events) {
+						t.FailNow()
+					}
+					t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+						events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
+				})
 			}
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
-		})
-	}
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch")
-
-	for _, metricSet := range metricSets {
-		t.Run(metricSet, func(t *testing.T) {
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet))
-			err := mbtest.WriteEventsReporterV2(f, t, metricSet)
-			if err != nil {
-				t.Fatal("write", err)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			for _, metricSet := range metricSets {
+				t.Run(metricSet, func(t *testing.T) {
+					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
+					err := mbtest.WriteEventsReporterV2(f, t, metricSet)
+					if err != nil {
+						t.Fatal("write", err)
+					}
+				})
 			}
-		})
-	}
-}
-
-// GetEnvHost returns host for Elasticsearch
-func getEnvHost() string {
-	host := os.Getenv("ES_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvPort returns port for Elasticsearch
-func getEnvPort() string {
-	port := os.Getenv("ES_PORT")
-
-	if len(port) == 0 {
-		port = "9200"
-	}
-	return port
+		},
+	})
 }
 
 // GetConfig returns config for elasticsearch module
-func getConfig(metricset string) map[string]interface{} {
+func getConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "elasticsearch",
 		"metricsets": []string{metricset},
-		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":      []string{host},
 		"index_recovery.active_only": false,
 	}
 }

From a322c5f05dfa0a606f47cbeb1a6eb5278987b9af Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:24:41 +0200
Subject: [PATCH 34/88] Run envoyproxy tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../server/server_integration_test.go         | 66 +++++++------------
 2 files changed, 26 insertions(+), 42 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 180299e81a71..71a07c022307 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -57,6 +57,8 @@ services:
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
+    ports:
+      - 9901
 
   etcd:
     build: ./module/etcd/_meta
diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go
index af001de45947..385732d257ca 100644
--- a/metricbeat/module/envoyproxy/server/server_integration_test.go
+++ b/metricbeat/module/envoyproxy/server/server_integration_test.go
@@ -20,7 +20,6 @@
 package server
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "envoyproxy")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "envoyproxy")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestEnvoyproxy(t *testing.T) {
+	runner := compose.TestRunner{Service: "envoyproxy"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "envoyproxy",
 		"metricsets": []string{"server"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ENVOYPROXY_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ENVOYPROXY_PORT")
-
-	if len(port) == 0 {
-		port = "9901"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 476530aedc83fe711c945c4ee33e8c1a86e79162 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:33:27 +0200
Subject: [PATCH 35/88] Run etcd tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../etcd/leader/leader_integration_test.go    | 66 +++++++------------
 .../module/etcd/self/self_integration_test.go | 66 +++++++------------
 .../etcd/store/store_integration_test.go      | 66 +++++++------------
 4 files changed, 74 insertions(+), 126 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 71a07c022307..594aed0129c1 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -62,6 +62,8 @@ services:
 
   etcd:
     build: ./module/etcd/_meta
+    ports:
+      - 2379
 
   haproxy:
     build: ./module/haproxy/_meta
diff --git a/metricbeat/module/etcd/leader/leader_integration_test.go b/metricbeat/module/etcd/leader/leader_integration_test.go
index 8ffee3e5934e..bd2dd7bc519c 100644
--- a/metricbeat/module/etcd/leader/leader_integration_test.go
+++ b/metricbeat/module/etcd/leader/leader_integration_test.go
@@ -20,7 +20,6 @@
 package leader
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestLeader(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"leader"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/etcd/self/self_integration_test.go b/metricbeat/module/etcd/self/self_integration_test.go
index 031d6e92c583..be8b979f7c13 100644
--- a/metricbeat/module/etcd/self/self_integration_test.go
+++ b/metricbeat/module/etcd/self/self_integration_test.go
@@ -20,7 +20,6 @@
 package self
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestLeader(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"self"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/etcd/store/store_integration_test.go b/metricbeat/module/etcd/store/store_integration_test.go
index 5b4c49f34c98..22be4797e8c1 100644
--- a/metricbeat/module/etcd/store/store_integration_test.go
+++ b/metricbeat/module/etcd/store/store_integration_test.go
@@ -20,7 +20,6 @@
 package store
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,51 +28,34 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "etcd")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func TestStore(t *testing.T) {
+	runner := compose.TestRunner{Service: "etcd"}
+
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
 		"metricsets": []string{"store"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("ETCD_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("ETCD_PORT")
-
-	if len(port) == 0 {
-		port = "2379"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 46bc0897ce9ee40998c39ba92a39ec914e26dc11 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:39:36 +0200
Subject: [PATCH 36/88] Run jolokia tests with testrunner

---
 metricbeat/docker-compose.yml                 |  3 +-
 .../jolokia/jmx/jmx_integration_test.go       | 76 +++++++------------
 2 files changed, 31 insertions(+), 48 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 594aed0129c1..2fe633f09b49 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -32,7 +32,6 @@ services:
       args:
         APACHE_VERSION: 2.4.12
 
-
   ceph:
     build: ./module/ceph/_meta
 
@@ -83,6 +82,8 @@ services:
 
   jolokia:
     build: ./module/jolokia/_meta
+    ports:
+      - 8778
 
   kafka:
     extends:
diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
index e66cd516bda3..d28146144e68 100644
--- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go
+++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
@@ -20,7 +20,6 @@
 package jmx
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,40 +28,41 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "jolokia")
+func TestJMX(t *testing.T) {
+	runner := compose.TestRunner{Service: "jolokia"}
 
-	for _, config := range getConfigs() {
-		f := mbtest.NewEventsFetcher(t, config)
-		events, err := f.Fetch()
-		if !assert.NoError(t, err) {
-			t.FailNow()
-		}
-		t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
-		if len(events) == 0 || len(events[0]) <= 1 {
-			t.Fatal("Empty events")
-		}
-	}
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "jolokia")
-
-	for _, config := range getConfigs() {
-		f := mbtest.NewEventsFetcher(t, config)
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			for _, config := range getConfigs(r.Host()) {
+				f := mbtest.NewEventsFetcher(t, config)
+				events, err := f.Fetch()
+				if !assert.NoError(t, err) {
+					t.FailNow()
+				}
+				t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
+				if len(events) == 0 || len(events[0]) <= 1 {
+					t.Fatal("Empty events")
+				}
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			for _, config := range getConfigs(r.Host()) {
+				f := mbtest.NewEventsFetcher(t, config)
+				err := mbtest.WriteEvents(f, t)
+				if err != nil {
+					t.Fatal("write", err)
+				}
+			}
+		},
+	})
 }
 
-func getConfigs() []map[string]interface{} {
+func getConfigs(host string) []map[string]interface{} {
 	return []map[string]interface{}{
 		{
 			"module":     "jolokia",
 			"metricsets": []string{"jmx"},
-			"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+			"hosts":      []string{host},
 			"namespace":  "testnamespace",
 			"jmx.mappings": []map[string]interface{}{
 				{
@@ -105,7 +105,7 @@ func getConfigs() []map[string]interface{} {
 		{
 			"module":     "jolokia",
 			"metricsets": []string{"jmx"},
-			"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
+			"hosts":      []string{host},
 			"namespace":  "testnamespace",
 			"jmx.mappings": []map[string]interface{}{
 				{
@@ -153,21 +153,3 @@ func getConfigs() []map[string]interface{} {
 		},
 	}
 }
-
-func getEnvHost() string {
-	host := os.Getenv("JOLOKIA_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("JOLOKIA_PORT")
-
-	if len(port) == 0 {
-		port = "8778"
-	}
-	return port
-}

From 4a036b72a716956353fcdc5b3ba1440f90b5d807 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 15:51:10 +0200
Subject: [PATCH 37/88] Run http tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../module/http/json/json_integration_test.go | 85 +++++++------------
 2 files changed, 34 insertions(+), 53 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 2fe633f09b49..be11942971fd 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -79,6 +79,8 @@ services:
 
   http:
     build: ./module/http/_meta
+    ports:
+      - 8080
 
   jolokia:
     build: ./module/jolokia/_meta
diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go
index cd215316263a..1e91c5ded020 100644
--- a/metricbeat/module/http/json/json_integration_test.go
+++ b/metricbeat/module/http/json/json_integration_test.go
@@ -20,7 +20,6 @@
 package json
 
 import (
-	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -29,41 +28,39 @@ import (
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestFetchObject(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("object"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestFetchArray(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("array"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("object"))
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-
+func TestJSON(t *testing.T) {
+	runner := compose.TestRunner{Service: "http"}
+
+	runner.Run(t, compose.Suite{
+		"FetchObject": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"FetchArray": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig(jsonType string) map[string]interface{} {
+func getConfig(jsonType string, host string) map[string]interface{} {
 	var path string
 	var responseIsArray bool
 	switch jsonType {
@@ -78,27 +75,9 @@ func getConfig(jsonType string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":        "http",
 		"metricsets":    []string{"json"},
-		"hosts":         []string{getEnvHost() + ":" + getEnvPort()},
+		"hosts":         []string{host},
 		"path":          path,
 		"namespace":     "testnamespace",
 		"json.is_array": responseIsArray,
 	}
 }
-
-func getEnvHost() string {
-	host := os.Getenv("HTTP_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("HTTP_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
-}

From 084e28ea1444da6ad8d948bb8253315a592fea5f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:11:59 +0200
Subject: [PATCH 38/88] Run kibana tests with testrunner

---
 metricbeat/docker-compose.yml                 |  4 ++
 metricbeat/module/kibana/mtest/testing.go     | 30 ++++---------
 .../kibana/stats/stats_integration_test.go    | 43 ++++++++++---------
 .../kibana/status/status_integration_test.go  | 39 ++++++++---------
 4 files changed, 53 insertions(+), 63 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index be11942971fd..53ec26cc66c3 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -110,6 +110,10 @@ services:
 
   kibana:
     build: ./module/kibana/_meta
+    depends_on:
+      - elasticsearch
+    ports:
+      - 5601
 
   #kubernetes:
   #  build: ./module/kubernetes/_meta
diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go
index a6dbc1f2f7e6..a8f1d191f80d 100644
--- a/metricbeat/module/kibana/mtest/testing.go
+++ b/metricbeat/module/kibana/mtest/testing.go
@@ -18,35 +18,21 @@
 package mtest
 
 import (
-	"net"
-	"os"
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// GetEnvHost returns host for Kibana
-func GetEnvHost() string {
-	host := os.Getenv("KIBANA_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
+var (
+	Runner = compose.TestRunner{
+		Service:  "kibana",
+		Parallel: true,
 	}
-	return host
-}
-
-// GetEnvPort returns port for Kibana
-func GetEnvPort() string {
-	port := os.Getenv("KIBANA_PORT")
-
-	if len(port) == 0 {
-		port = "5601"
-	}
-	return port
-}
+)
 
 // GetConfig returns config for kibana module
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kibana",
 		"metricsets": []string{metricset},
-		"hosts":      []string{net.JoinHostPort(GetEnvHost(), GetEnvPort())},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go
index eb5eb3566c0c..f52ffadca7cd 100644
--- a/metricbeat/module/kibana/stats/stats_integration_test.go
+++ b/metricbeat/module/kibana/stats/stats_integration_test.go
@@ -33,30 +33,31 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kibana/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "kibana")
+func TestStat(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			config := mtest.GetConfig("stats", r.Host())
+			version, err := getKibanaVersion(r.Host())
+			if err != nil {
+				t.Fatal("getting kibana version", err)
+			}
 
-	config := mtest.GetConfig("stats")
-	host := config["hosts"].([]string)[0]
-	version, err := getKibanaVersion(host)
-	if err != nil {
-		t.Fatal("getting kibana version", err)
-	}
-
-	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-	if err != nil {
-		t.Fatal("checking if kibana stats API is available", err)
-	}
+			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+			if err != nil {
+				t.Fatal("checking if kibana stats API is available", err)
+			}
 
-	if !isStatsAPIAvailable {
-		t.Skip("Kibana stats API is not available until 6.4.0")
-	}
+			if !isStatsAPIAvailable {
+				t.Skip("Kibana stats API is not available until 6.4.0")
+			}
 
-	f := mbtest.NewReportingMetricSetV2(t, config)
-	err = mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			f := mbtest.NewReportingMetricSetV2(t, config)
+			err = mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
 func getKibanaVersion(kibanaHostPort string) (string, error) {
diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go
index 7bf989118cb2..34013a4ad532 100644
--- a/metricbeat/module/kibana/status/status_integration_test.go
+++ b/metricbeat/module/kibana/status/status_integration_test.go
@@ -29,24 +29,23 @@ import (
 	"github.com/elastic/beats/metricbeat/module/kibana/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUpWithTimeout(t, 600, "elasticsearch", "kibana")
-
-	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "elasticsearch", "kibana")
-
-	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 9d93da7d6ae248cc863ff2e4bc547823242dab8f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:30:37 +0200
Subject: [PATCH 39/88] Run logstash tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../logstash/node/node_integration_test.go    | 42 +++++++++----------
 .../node_stats/node_stats_integration_test.go | 42 +++++++++----------
 metricbeat/module/logstash/testing.go         | 31 +++++---------
 4 files changed, 54 insertions(+), 63 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 53ec26cc66c3..d8bf286947ad 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -135,6 +135,8 @@ services:
 
   logstash:
     build: ./module/logstash/_meta
+    ports:
+      - 9600
 
   memcached:
     build: ./module/memcached/_meta
diff --git a/metricbeat/module/logstash/node/node_integration_test.go b/metricbeat/module/logstash/node/node_integration_test.go
index 6ec042115e18..891020ae46eb 100644
--- a/metricbeat/module/logstash/node/node_integration_test.go
+++ b/metricbeat/module/logstash/node/node_integration_test.go
@@ -29,25 +29,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNode(t *testing.T) {
+	logstash.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+
+		},
+	})
 }
diff --git a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go b/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
index 0883d629c1f7..492694e70f1c 100644
--- a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
+++ b/metricbeat/module/logstash/node_stats/node_stats_integration_test.go
@@ -29,25 +29,25 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats"))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "logstash")
-
-	f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats"))
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNodeStats(t *testing.T) {
+	logstash.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, logstash.GetConfig("node_stats", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+
+		},
+	})
 }
diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go
index 221b280bda04..f262b02bc9aa 100644
--- a/metricbeat/module/logstash/testing.go
+++ b/metricbeat/module/logstash/testing.go
@@ -17,33 +17,22 @@
 
 package logstash
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-// GetEnvHost for Logstash
-func GetEnvHost() string {
-	host := os.Getenv("LOGSTASH_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvPort for Logstash
-func GetEnvPort() string {
-	port := os.Getenv("LOGSTASH_PORT")
-
-	if len(port) == 0 {
-		port = "9600"
+var (
+	Runner = compose.TestRunner{
+		Service:  "logstash",
+		Parallel: true,
 	}
-	return port
-}
+)
 
 // GetConfig for Logstash
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "logstash",
 		"metricsets": []string{metricset},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
+		"hosts":      []string{host},
 	}
 }

From 9368483ad7fce13769b07952908fdfa01f7a6a3f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 16:50:39 +0200
Subject: [PATCH 40/88] Skip tests using EnsureUp by now

---
 metricbeat/module/memcached/stats/stats_integration_test.go     | 1 +
 .../module/mongodb/collstats/collstats_integration_test.go      | 2 ++
 metricbeat/module/mongodb/dbstats/dbstats_integration_test.go   | 2 ++
 metricbeat/module/mongodb/metrics/metrics_intergration_test.go  | 2 ++
 .../module/mongodb/replstatus/replstatus_integration_test.go    | 2 ++
 metricbeat/module/mongodb/status/status_integration_test.go     | 2 ++
 metricbeat/module/mysql/mysql_integration_test.go               | 1 +
 metricbeat/module/mysql/status/status_integration_test.go       | 2 ++
 .../module/nginx/stubstatus/stubstatus_integration_test.go      | 1 +
 metricbeat/module/php_fpm/pool/pool_integration_test.go         | 1 +
 .../module/postgresql/activity/activity_integration_test.go     | 2 ++
 .../module/postgresql/bgwriter/bgwriter_integration_test.go     | 2 ++
 .../module/postgresql/database/database_integration_test.go     | 2 ++
 .../module/postgresql/statement/statement_integration_test.go   | 2 ++
 .../module/prometheus/collector/collector_integration_test.go   | 2 ++
 metricbeat/module/prometheus/stats/stats_integration_test.go    | 2 ++
 .../module/rabbitmq/connection/connection_integration_test.go   | 1 +
 .../module/rabbitmq/exchange/exchange_integration_test.go       | 1 +
 metricbeat/module/rabbitmq/node/node_integration_test.go        | 1 +
 metricbeat/module/rabbitmq/queue/queue_integration_test.go      | 1 +
 metricbeat/module/traefik/health/health_integration_test.go     | 2 ++
 metricbeat/module/uwsgi/status/status_integration_test.go       | 2 ++
 metricbeat/module/zookeeper/mntr/mntr_integration_test.go       | 2 ++
 23 files changed, 38 insertions(+)

diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index a195194815a1..112033e08d6e 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "memcached")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 0de42bd027f1..4f860e98e8ed 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -51,6 +52,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 512ef1c2b8ed..75fc73790ab9 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -76,6 +77,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index 1347cd02874f..f39b950846f4 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -49,6 +50,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index 40e60bca51ba..526a81aba8fb 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -34,6 +34,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	err := initiateReplicaSet(t)
@@ -73,6 +74,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 4b3969979d95..2140b0c181be 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -53,6 +54,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mongodb")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 772fd91ae80d..32b90a519b62 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestNewDB(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	db, err := NewDB(GetMySQLEnvDSN())
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index 7ccc0a77ef5c..e0a4898de35e 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	f := mbtest.NewEventFetcher(t, getConfig(false))
@@ -55,6 +56,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestFetchRaw(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "mysql")
 
 	f := mbtest.NewEventFetcher(t, getConfig(true))
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index d4b128870032..779e400f02bc 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "nginx")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index aaac9336929c..f779ad41f0c4 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "phpfpm")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index d4467d1b5460..0bc64f1de549 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -58,6 +59,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 	f := mbtest.NewEventsFetcher(t, getConfig())
 
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 72b6bed4f601..04203eee0c5e 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -60,6 +61,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index 50287dce897d..e09dd7077274 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -60,6 +61,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index 58938df7c6be..b3449ed5c967 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -91,6 +92,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "postgresql")
 	f := mbtest.NewEventsFetcher(t, getConfig())
 
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index c09ac37f27a9..d2f186e9d064 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -33,6 +33,7 @@ import (
 // Every prometheus exporter should work here.
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
@@ -45,6 +46,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 13154a351b6f..45e62466ec0e 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -47,6 +48,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "prometheus")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index 26597e7c36d6..ae0f7a938685 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index 0903a7bb035b..b8094935b8c1 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index 48c05f1d6965..938ac90024ec 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -28,6 +28,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index ccff728cf2ff..9b348ac0a376 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -29,6 +29,7 @@ import (
 )
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "rabbitmq")
 
 	f := mbtest.NewEventsFetcher(t, getConfig())
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index 388921bdd3d2..8a49f755fa1f 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -42,6 +42,7 @@ func makeBadRequest(config map[string]interface{}) error {
 }
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "traefik")
 
 	config := mtest.GetConfig("health")
@@ -66,6 +67,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "traefik")
 
 	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health"))
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 5ac48b5e4a2d..248b4612f478 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -30,6 +30,7 @@ import (
 )
 
 func TestFetchTCP(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "uwsgi_tcp")
 
 	f := mbtest.NewEventsFetcher(t, getConfig("tcp"))
@@ -42,6 +43,7 @@ func TestFetchTCP(t *testing.T) {
 }
 
 func TestFetchHTTP(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "uwsgi_http")
 
 	f := mbtest.NewEventsFetcher(t, getConfig("http"))
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index 7828c4e6e428..2f1624358c00 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -31,6 +31,7 @@ import (
 )
 
 func TestFetch(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "zookeeper")
 
 	f := mbtest.NewEventFetcher(t, getConfig())
@@ -57,6 +58,7 @@ func TestFetch(t *testing.T) {
 }
 
 func TestData(t *testing.T) {
+	t.Skip("ignoring tests with EnsureUp by now")
 	compose.EnsureUp(t, "zookeeper")
 
 	f := mbtest.NewEventFetcher(t, getConfig())

From 26e33a6bd4fb5759d46102a0e4380d0ac2c00921 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:25:02 +0200
Subject: [PATCH 41/88] Fix environment setting

---
 libbeat/tests/compose/runner.go  | 1 -
 libbeat/tests/compose/wrapper.go | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 4484e9b616e6..39a678043e3d 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -116,7 +116,6 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	for _, s := range scenarios {
 		var vars []string
 		for k, v := range s {
-			os.Setenv(k, v)
 			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
 		}
 		sort.Strings(vars)
diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go
index 119b68280c39..7a5997f62591 100644
--- a/libbeat/tests/compose/wrapper.go
+++ b/libbeat/tests/compose/wrapper.go
@@ -117,7 +117,7 @@ func (d *wrapperDriver) cmd(ctx context.Context, command string, arg ...string)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	if len(d.Environment) > 0 {
-		cmd.Env = append(d.Environment, os.Environ()...)
+		cmd.Env = append(os.Environ(), d.Environment...)
 	}
 	return cmd
 }

From fb972953670bb490e32a3d31d2b69cde87c524b6 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:21:47 +0200
Subject: [PATCH 42/88] Test elasticsearch on multiple versions

---
 metricbeat/docker-compose.yml                 | 12 +++---------
 .../module/elasticsearch/_meta/Dockerfile     |  2 --
 .../module/elasticsearch/docker-compose.yml   | 19 +++++++++++++++++++
 .../elasticsearch_integration_test.go         | 13 ++++++++++++-
 4 files changed, 34 insertions(+), 12 deletions(-)
 delete mode 100644 metricbeat/module/elasticsearch/_meta/Dockerfile
 create mode 100644 metricbeat/module/elasticsearch/docker-compose.yml

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d8bf286947ad..ce2e79c05c22 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -44,15 +44,9 @@ services:
       - 8080
 
   elasticsearch:
-    build: ./module/elasticsearch/_meta
-    environment:
-      - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
-      - "network.host="
-      - "transport.host=127.0.0.1"
-      - "http.host=0.0.0.0"
-      - "xpack.security.enabled=false"
-    ports:
-      - 9200
+    extends:
+      file: ./module/elasticsearch/docker-compose.yml
+      service: elasticsearch
 
   envoyproxy:
     build: ./module/envoyproxy/_meta
diff --git a/metricbeat/module/elasticsearch/_meta/Dockerfile b/metricbeat/module/elasticsearch/_meta/Dockerfile
deleted file mode 100644
index 4de235037102..000000000000
--- a/metricbeat/module/elasticsearch/_meta/Dockerfile
+++ /dev/null
@@ -1,2 +0,0 @@
-FROM docker.elastic.co/elasticsearch/elasticsearch:6.3.0
-HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:9200
diff --git a/metricbeat/module/elasticsearch/docker-compose.yml b/metricbeat/module/elasticsearch/docker-compose.yml
new file mode 100644
index 000000000000..91828821c33d
--- /dev/null
+++ b/metricbeat/module/elasticsearch/docker-compose.yml
@@ -0,0 +1,19 @@
+version: "2.1"
+
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.3.0}
+    environment:
+      - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
+      - "network.host="
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      - "xpack.security.enabled=false"
+    ports:
+      - 9200
+    healthcheck:
+      test:
+      - "CMD-SHELL"
+      - "curl -f http://localhost:9200"
+      interval: 1s
+      retries: 300
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 2345cd5f07f1..00bc0b3db86c 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -54,7 +54,18 @@ var metricSets = []string{
 }
 
 func TestElasticsearch(t *testing.T) {
-	runner := compose.TestRunner{Service: "elasticsearch"}
+	runner := compose.TestRunner{
+		Service: "elasticsearch",
+		Options: compose.RunnerOptions{
+			"ELASTICSEARCH_VERSION": {
+				"6.4.0",
+				"6.3.2",
+				// "6.2.4",
+				"5.6.11",
+			},
+		},
+		Parallel: true,
+	}
 
 	runner.Run(t, compose.Suite{
 		"Fetch": func(t *testing.T, r compose.R) {

From ba2cac40ff14abe37ad4db64b8914ab6c158277a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 11 Sep 2018 18:37:58 +0200
Subject: [PATCH 43/88] Expose options to tests

---
 libbeat/tests/compose/runner.go                          | 9 +++++++++
 .../elasticsearch/elasticsearch_integration_test.go      | 6 +++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 39a678043e3d..0ce16f3e1851 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -114,6 +114,7 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		t.Fatal("Test runner configuration doesn't produce scenarios")
 	}
 	for _, s := range scenarios {
+		s := s
 		var vars []string
 		for k, v := range s {
 			vars = append(vars, fmt.Sprintf("%s=%s", k, v))
@@ -167,6 +168,10 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 
 type R interface {
 	Host() string
+	Hostname() string
+	Port() string
+
+	Option(string) string
 }
 
 type runnerControl struct {
@@ -188,3 +193,7 @@ func (r *runnerControl) Port() string {
 	_, port, _ := net.SplitHostPort(r.host)
 	return port
 }
+
+func (r *runnerControl) Option(key string) string {
+	return r.scenario[key]
+}
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 00bc0b3db86c..2126db2283a5 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -60,7 +60,7 @@ func TestElasticsearch(t *testing.T) {
 			"ELASTICSEARCH_VERSION": {
 				"6.4.0",
 				"6.3.2",
-				// "6.2.4",
+				"6.2.4",
 				"5.6.11",
 			},
 		},
@@ -69,6 +69,10 @@ func TestElasticsearch(t *testing.T) {
 
 	runner.Run(t, compose.Suite{
 		"Fetch": func(t *testing.T, r compose.R) {
+			if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+				t.Skip("This test fails on this version")
+			}
+
 			host := r.Host()
 			err := createIndex(host)
 			assert.NoError(t, err)

From 44a8772a8800bb829065ef5e3fc0036ff225ffdb Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 11:35:30 +0200
Subject: [PATCH 44/88] Fix kafka python tests

---
 metricbeat/docker-compose.yml            | 3 +++
 metricbeat/module/kafka/_meta/Dockerfile | 1 -
 metricbeat/module/kafka/_meta/run.sh     | 5 +++++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index ce2e79c05c22..10c4e8d0780e 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -85,6 +85,9 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
 
   kafka_1_1_0:
     extends:
diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile
index aab10dbed24b..b16e4a4594ef 100644
--- a/metricbeat/module/kafka/_meta/Dockerfile
+++ b/metricbeat/module/kafka/_meta/Dockerfile
@@ -3,7 +3,6 @@ FROM debian:stretch
 ARG KAFKA_VERSION=2.0.0
 
 ENV KAFKA_HOME /kafka
-# The advertised host is kafka. This means it will not work if container is started locally and connected from localhost to it
 ENV KAFKA_LOGS_DIR="/kafka-logs"
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 696b9507b4bc..23f085e8c646 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -1,5 +1,10 @@
 #!/bin/bash
 
+if [ -n "$KAFKA_ADVERTISED_HOST_AUTO" ]; then
+	ip=$(nslookup $HOSTNAME 2> /dev/null | grep ^Address | cut -d' ' -f3)
+	KAFKA_ADVERTISED_HOST=${ip}:9092
+fi
+
 # Check if KAFKA_ADVERTISED_HOST is set
 # if not wait to read it from file
 if [ -z "$KAFKA_ADVERTISED_HOST" ]; then

From 6881d7bee66834a8a1af37caf233d899bbedb935 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 12:24:12 +0200
Subject: [PATCH 45/88] Use alpine jre image for kafka

---
 metricbeat/module/kafka/_meta/Dockerfile | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile
index b16e4a4594ef..3be204a85455 100644
--- a/metricbeat/module/kafka/_meta/Dockerfile
+++ b/metricbeat/module/kafka/_meta/Dockerfile
@@ -1,4 +1,4 @@
-FROM debian:stretch
+FROM openjdk:8-jre-alpine3.8
 
 ARG KAFKA_VERSION=2.0.0
 
@@ -7,11 +7,12 @@ ENV KAFKA_LOGS_DIR="/kafka-logs"
 ENV _JAVA_OPTIONS "-Djava.net.preferIPv4Stack=true"
 ENV TERM=linux
 
-RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils
+RUN apk add -u bash
 
-RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \
-    "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
-    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1
+RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && \
+    wget -q -O $INSTALL_DIR/kafka.tgz "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \
+    tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 && \
+    rm -f ${INSTALL_DIR}/kafka.tgz
 
 ADD run.sh /run.sh
 ADD healthcheck.sh /healthcheck.sh

From eb3c9fe5be413fb794cc3fd96d0d5d61dfccb888 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 12:43:47 +0200
Subject: [PATCH 46/88] Actually test kafka with multiple versions, and reuse
 images

---
 metricbeat/docker-compose.yml              | 8 ++++++++
 metricbeat/module/kafka/docker-compose.yml | 3 ++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 10c4e8d0780e..d38de2ad12a9 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -93,6 +93,10 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
+    image: metricbeat-kafka:1.1.0
     build:
       args:
         KAFKA_VERSION: 1.1.0
@@ -101,6 +105,10 @@ services:
     extends:
       file: ./module/kafka/docker-compose.yml
       service: kafka
+    environment:
+      # Needed for python tests
+      KAFKA_ADVERTISED_HOST_AUTO: "yes"
+    image: metricbeat-kafka:0.10.2.1
     build:
       args:
         KAFKA_VERSION: 0.10.2.1
diff --git a/metricbeat/module/kafka/docker-compose.yml b/metricbeat/module/kafka/docker-compose.yml
index abd3e8f0aeda..b28bf74daeeb 100644
--- a/metricbeat/module/kafka/docker-compose.yml
+++ b/metricbeat/module/kafka/docker-compose.yml
@@ -2,9 +2,10 @@ version: "2.1"
 
 services:
   kafka:
+    image: metricbeat-kafka:${KAFKA_VERSION:-2.0.0}
     build:
       context: ./_meta
       args:
-        KAFKA_VERSION: 2.0.0
+        KAFKA_VERSION: ${KAFKA_VERSION:-2.0.0}
     ports:
     - 9092

From 22367be99abedcd44be1b4f1339d6c3014934c78 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 15:46:09 +0200
Subject: [PATCH 47/88] Fix golang python test

---
 metricbeat/tests/system/test_golang.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_golang.py b/metricbeat/tests/system/test_golang.py
index a053a2403fa0..fe07f28f5db6 100644
--- a/metricbeat/tests/system/test_golang.py
+++ b/metricbeat/tests/system/test_golang.py
@@ -11,7 +11,7 @@ class Test(metricbeat.BaseTest):
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     def test_stats(self):
         """
-        prometheus stats test
+        golang heap test
         """
         self.render_config_template(modules=[{
             "name": "golang",
@@ -43,5 +43,5 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('GOLANG_HOST', self.compose_hosts()[0]) + ':' +
+        return ["http://" + os.getenv('GOLANG_HOST', 'localhost') + ':' +
                 os.getenv('GOLANG_PORT', '6060')]

From b3f09e0451109343d5d0829d83d6759ae554e447 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 16:11:40 +0200
Subject: [PATCH 48/88] Fix kibana python test

---
 metricbeat/tests/system/test_kibana.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index f3143819884f..456588201ae2 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,7 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', self.compose_hosts()[1]) + ':' +
+        return [os.getenv('KIBANA_HOST', self.compose_hosts()[0]) + ':' +
                 os.getenv('KIBANA_PORT', '5601')]
 
     def get_version(self):

From 3fcd7b82ce874be7be0f2f812604a9a953a84191 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 16:37:06 +0200
Subject: [PATCH 49/88] Fix test_base python test

---
 metricbeat/tests/system/test_base.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py
index 361d51a466e7..770ea0ef7160 100644
--- a/metricbeat/tests/system/test_base.py
+++ b/metricbeat/tests/system/test_base.py
@@ -76,3 +76,18 @@ def test_dashboards(self):
 
         assert exit_code == 0
         assert self.log_contains("Kibana dashboards successfully loaded.")
+
+    def get_elasticsearch_url(self):
+        return "http://{host}:{port}".format(
+            host=os.getenv("ES_HOST", self.compose_hosts()[1]),
+            port=os.getenv("ES_PORT", "9200"),
+        )
+
+    def get_kibana_url(self):
+        """
+        Returns kibana host URL
+        """
+        return "http://{host}:{port}".format(
+            host=os.getenv("KIBANA_HOST", self.compose_hosts()[0]),
+            port=os.getenv("KIBANA_PORT", "5601"),
+        )

From fcfb66354fd2269137d6e038882e166811c9d756 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 18:45:47 +0200
Subject: [PATCH 50/88] Fix docker python test

---
 metricbeat/tests/system/test_docker.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/metricbeat/tests/system/test_docker.py b/metricbeat/tests/system/test_docker.py
index df0e615e8743..616e7627604f 100644
--- a/metricbeat/tests/system/test_docker.py
+++ b/metricbeat/tests/system/test_docker.py
@@ -6,6 +6,7 @@
 
 
 class Test(metricbeat.BaseTest):
+    COMPOSE_SERVICES = ["redis"]  # Just to have a container running
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     def test_container_fields(self):

From f0f995430d965b5485807bdd113f7aa5f1da927c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 16:45:03 +0200
Subject: [PATCH 51/88] Generate data in kafka tests before starting consumer

---
 .../consumergroup_integration_test.go         | 12 +---
 metricbeat/module/kafka/mtest/helpers.go      | 72 +++++++++++++++++++
 .../partition/partition_integration_test.go   | 45 +-----------
 3 files changed, 78 insertions(+), 51 deletions(-)
 create mode 100644 metricbeat/module/kafka/mtest/helpers.go

diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index 8a6541057c7e..e5a3e1f20b79 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -20,11 +20,9 @@
 package consumergroup
 
 import (
-	"io"
 	"testing"
 	"time"
 
-	saramacluster "github.com/bsm/sarama-cluster"
 	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
@@ -34,7 +32,9 @@ import (
 
 func TestConsumerGroup(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		c, err := startConsumer(t, "metricbeat-test", r.Host())
+		topic := "metricbeat-test"
+		mtest.GenerateKafkaData(t, topic, r.Host())
+		c, err := mtest.StartConsumer(t, topic, r.Host())
 		if err != nil {
 			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
 		}
@@ -52,12 +52,6 @@ func TestConsumerGroup(t *testing.T) {
 	}})
 }
 
-func startConsumer(t *testing.T, topic, host string) (io.Closer, error) {
-	brokers := []string{host}
-	topics := []string{topic}
-	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
-}
-
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "kafka",
diff --git a/metricbeat/module/kafka/mtest/helpers.go b/metricbeat/module/kafka/mtest/helpers.go
new file mode 100644
index 000000000000..024784a210ec
--- /dev/null
+++ b/metricbeat/module/kafka/mtest/helpers.go
@@ -0,0 +1,72 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"io"
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	saramacluster "github.com/bsm/sarama-cluster"
+)
+
+func GenerateKafkaData(t *testing.T, topic string, host string) {
+	t.Logf("Send Kafka Event to topic: %v", topic)
+
+	config := sarama.NewConfig()
+	config.Producer.Return.Successes = true
+	// Retry for 10 seconds
+	config.Producer.Retry.Max = 20
+	config.Producer.Retry.Backoff = 500 * time.Millisecond
+	config.Metadata.Retry.Max = 20
+	config.Metadata.Retry.Backoff = 500 * time.Millisecond
+	client, err := sarama.NewClient([]string{host}, config)
+	if err != nil {
+		t.Errorf("%s", err)
+		t.FailNow()
+	}
+
+	producer, err := sarama.NewSyncProducerFromClient(client)
+	if err != nil {
+		t.Error(err)
+	}
+	defer producer.Close()
+
+	msg := &sarama.ProducerMessage{
+		Topic: topic,
+		Value: sarama.StringEncoder("Hello World"),
+	}
+
+	_, _, err = producer.SendMessage(msg)
+	if err != nil {
+		t.Errorf("failed to send message: %s\n", err)
+	}
+
+	err = client.RefreshMetadata(topic)
+	if err != nil {
+		t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err)
+	}
+}
+
+func StartConsumer(t *testing.T, topic, host string) (io.Closer, error) {
+	brokers := []string{host}
+	topics := []string{topic}
+
+	return saramacluster.NewConsumer(brokers, "test-group", topics, nil)
+}
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 3b0c8c507fad..1ef103a208f7 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -26,7 +26,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/Shopify/sarama"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -43,7 +42,7 @@ func TestPartition(t *testing.T) {
 
 	mtest.Runner.Run(t, compose.Suite{
 		"Data": func(t *testing.T, r compose.R) {
-			generateKafkaData(t, "metricbeat-generate-data", r.Host())
+			mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
 
 			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
 			err := mbtest.WriteEventsReporterV2(ms, t, "")
@@ -56,7 +55,7 @@ func TestPartition(t *testing.T) {
 			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
 
 			// Create initial topic
-			generateKafkaData(t, testTopic, r.Host())
+			mtest.GenerateKafkaData(t, testTopic, r.Host())
 
 			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
 			dataBefore, err := mbtest.ReportingFetchV2(f)
@@ -71,7 +70,7 @@ func TestPartition(t *testing.T) {
 			var n int64 = 10
 			// Create n messages
 			for i := int64(0); i < n; i++ {
-				generateKafkaData(t, testTopic, r.Host())
+				mtest.GenerateKafkaData(t, testTopic, r.Host())
 			}
 
 			dataAfter, err := mbtest.ReportingFetchV2(f)
@@ -111,44 +110,6 @@ func TestPartition(t *testing.T) {
 		}})
 }
 
-func generateKafkaData(t *testing.T, topic string, host string) {
-	t.Logf("Send Kafka Event to topic: %v", topic)
-
-	config := sarama.NewConfig()
-	config.Producer.Return.Successes = true
-	// Retry for 10 seconds
-	config.Producer.Retry.Max = 20
-	config.Producer.Retry.Backoff = 500 * time.Millisecond
-	config.Metadata.Retry.Max = 20
-	config.Metadata.Retry.Backoff = 500 * time.Millisecond
-	client, err := sarama.NewClient([]string{host}, config)
-	if err != nil {
-		t.Errorf("%s", err)
-		t.FailNow()
-	}
-
-	producer, err := sarama.NewSyncProducerFromClient(client)
-	if err != nil {
-		t.Error(err)
-	}
-	defer producer.Close()
-
-	msg := &sarama.ProducerMessage{
-		Topic: topic,
-		Value: sarama.StringEncoder("Hello World"),
-	}
-
-	_, _, err = producer.SendMessage(msg)
-	if err != nil {
-		t.Errorf("failed to send message: %s\n", err)
-	}
-
-	err = client.RefreshMetadata(topic)
-	if err != nil {
-		t.Errorf("failed to refresh metadata for topic '%s': %s\n", topic, err)
-	}
-}
-
 func getConfig(topic string, host string) map[string]interface{} {
 	var topics []string
 	if topic != "" {

From 6c0f666db46791b9bfac634b26e02c0cd6a479b5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 18:41:57 +0200
Subject: [PATCH 52/88] Reduce number of kafka partitions in tests

---
 metricbeat/module/kafka/_meta/run.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/metricbeat/module/kafka/_meta/run.sh b/metricbeat/module/kafka/_meta/run.sh
index 23f085e8c646..0d845b8cb537 100755
--- a/metricbeat/module/kafka/_meta/run.sh
+++ b/metricbeat/module/kafka/_meta/run.sh
@@ -47,6 +47,8 @@ ${KAFKA_HOME}/bin/kafka-server-start.sh ${KAFKA_HOME}/config/server.properties \
     --override advertised.listeners=INSIDE://localhost:9091,OUTSIDE://$KAFKA_ADVERTISED_HOST \
     --override listener.security.protocol.map=INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT \
     --override inter.broker.listener.name=INSIDE \
+    --override offsets.topic.replication.factor=1 \
+    --override offsets.topic.num.partitions=2 \
     --override logs.dir=${KAFKA_LOGS_DIR} &
 
 wait_for_port 9092

From c14c93eebe1f0cf19ef730546851afd89acc7b3a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 13 Sep 2018 20:33:11 +0200
Subject: [PATCH 53/88] Don't prebuild docker images

---
 libbeat/scripts/Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
index 2c8d370242e4..1d307cb38ffc 100755
--- a/libbeat/scripts/Makefile
+++ b/libbeat/scripts/Makefile
@@ -371,14 +371,14 @@ import-dashboards: update ${BEAT_NAME}
 # Builds the environment to test beat
 .PHONY: build-image
 build-image: write-environment
-	${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
+	# ${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
 
 # Runs the environment so the redis and elasticsearch can also be used for local development
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.
 .PHONY: start-environment
 start-environment: stop-environment
 	${DOCKER_COMPOSE} pull --include-deps
-	${DOCKER_COMPOSE} up -d
+	${DOCKER_COMPOSE} up --build -d
 
 .PHONY: stop-environment
 stop-environment:

From 39d3b1bd5db6208795929d4bce89af54c89640a5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 12 Sep 2018 15:34:10 +0200
Subject: [PATCH 54/88] Fix uwsgi python test

---
 metricbeat/docker-compose.yml            |  6 +++-
 metricbeat/module/uwsgi/_meta/Dockerfile |  5 +---
 metricbeat/tests/system/test_uwsgi.py    | 36 +++++++++---------------
 3 files changed, 19 insertions(+), 28 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d38de2ad12a9..d2353db248e2 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -194,10 +194,14 @@ services:
   uwsgi_tcp:
     build: ./module/uwsgi/_meta
     command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --wsgi-file app.py
+    ports:
+      - 9191
 
   uwsgi_http:
     build: ./module/uwsgi/_meta
-    command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9192 --memory-report --stats-http --wsgi-file app.py
+    command: uwsgi --http :8080 --master --processes 1 --threads 2 --stats 0.0.0.0:9191 --memory-report --stats-http --wsgi-file app.py
+    ports:
+      - 9191
 
   zookeeper:
     build: ./module/zookeeper/_meta
diff --git a/metricbeat/module/uwsgi/_meta/Dockerfile b/metricbeat/module/uwsgi/_meta/Dockerfile
index 81d8372ec122..17fc9571bc9a 100644
--- a/metricbeat/module/uwsgi/_meta/Dockerfile
+++ b/metricbeat/module/uwsgi/_meta/Dockerfile
@@ -6,7 +6,4 @@ RUN pip install --no-cache-dir --trusted-host pypi.python.org uwsgi
 WORKDIR /app
 COPY testdata/app /app
 
-HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080/
-EXPOSE 8080 9191 9192
-
-CMD [""]
+HEALTHCHECK --interval=1s --retries=60 --timeout=10s CMD curl http://localhost:8080
diff --git a/metricbeat/tests/system/test_uwsgi.py b/metricbeat/tests/system/test_uwsgi.py
index 3707d2cab862..7c4bfd79ddea 100644
--- a/metricbeat/tests/system/test_uwsgi.py
+++ b/metricbeat/tests/system/test_uwsgi.py
@@ -8,8 +8,7 @@
 
 
 class Test(metricbeat.BaseTest):
-
-    COMPOSE_SERVICES = ['uwsgi_tcp', "uwsgi_http"]
+    COMPOSE_SERVICES = ['uwsgi_tcp']
 
     def common_checks(self, output):
         # Ensure no errors or warnings exist in the log.
@@ -51,15 +50,14 @@ def common_checks(self, output):
 
     @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
     @attr('integration')
-    def test_status_tcp(self):
+    def test_status(self):
         """
         uWSGI module outputs an event.
         """
-        hosts = [os.getenv("UWSGI_STAT_TCP_SERVER")]
         self.render_config_template(modules=[{
             "name": "uwsgi",
             "metricsets": ["status"],
-            "hosts": hosts,
+            "hosts": [self.get_host()],
             "period": "5s"
         }])
         proc = self.start_beat()
@@ -69,22 +67,14 @@ def test_status_tcp(self):
         output = self.read_output_json()
         self.common_checks(output)
 
-    @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test")
-    @attr('integration')
-    def test_status_http(self):
-        """
-        uWSGI module outputs an event.
-        """
-        hosts = [os.getenv("UWSGI_STAT_HTTP_SERVER")]
-        self.render_config_template(modules=[{
-            "name": "uwsgi",
-            "metricsets": ["status"],
-            "hosts": hosts,
-            "period": "5s"
-        }])
-        proc = self.start_beat()
-        self.wait_until(lambda: self.output_lines() > 0)
-        proc.check_kill_and_wait()
+    def get_host(self):
+        return os.getenv("UWSGI_STAT_TCP_SERVER",
+                         "tcp://{}:9191".format(self.compose_hosts()[0]))
 
-        output = self.read_output_json()
-        self.common_checks(output)
+
+class TestHTTP(Test):
+    COMPOSE_SERVICES = ['uwsgi_http']
+
+    def get_host(self):
+        return os.getenv("UWSGI_STAT_HTTP_SERVER",
+                         "http://{}:9191".format(self.compose_hosts()[0]))

From ab3a368c898384ac93398e895a2c620b97b75872 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:09:04 +0100
Subject: [PATCH 55/88] Run zookeeper tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../zookeeper/mntr/mntr_integration_test.go   | 65 +++++++++----------
 metricbeat/module/zookeeper/testing.go        | 48 --------------
 3 files changed, 34 insertions(+), 81 deletions(-)
 delete mode 100644 metricbeat/module/zookeeper/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d2353db248e2..e25330a02cdc 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -205,3 +205,5 @@ services:
 
   zookeeper:
     build: ./module/zookeeper/_meta
+    ports:
+      - 2181
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index 2f1624358c00..d73aa21b5fba 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -27,52 +27,51 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/zookeeper"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "zookeeper")
+func TestMntr(t *testing.T) {
+	runner := compose.TestRunner{Service: "zookeeper", Parallel: true}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	// Check values
-	version := event["version"].(string)
-	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
-	numAliveConnections := event["num_alive_connections"].(int64)
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
-	assert.True(t, avgLatency >= 0)
-	assert.True(t, maxLatency >= 0)
-	assert.True(t, numAliveConnections > 0)
+			// Check values
+			version := event["version"].(string)
+			avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
+			maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+			numAliveConnections := event["num_alive_connections"].(int64)
 
-	// Check number of fields. At least 10, depending on environment
-	assert.True(t, len(event) >= 10)
-}
+			assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
+			assert.True(t, avgLatency >= 0)
+			assert.True(t, maxLatency >= 0)
+			assert.True(t, numAliveConnections > 0)
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "zookeeper")
+			// Check number of fields. At least 10, depending on environment
+			assert.True(t, len(event) >= 10)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	f := mbtest.NewEventFetcher(t, getConfig())
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "zookeeper",
 		"metricsets": []string{"mntr"},
-		"hosts":      []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/zookeeper/testing.go b/metricbeat/module/zookeeper/testing.go
deleted file mode 100644
index afa3cac8398e..000000000000
--- a/metricbeat/module/zookeeper/testing.go
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package zookeeper
-
-import (
-	"os"
-)
-
-// Helper functions for testing used in the zookeeper MetricSets.
-
-// GetZookeeperEnvHost returns the hostname of the ZooKeeper server to use for
-// testing. It reads the value from the ZOOKEEPER_HOST environment variable and
-// returns localhost if it is not set.
-func GetZookeeperEnvHost() string {
-	host := os.Getenv("ZOOKEEPER_HOST")
-
-	if len(host) == 0 {
-		host = "localhost"
-	}
-	return host
-}
-
-// GetZookeeperEnvPort returns the port of the ZooKeeper server to use for
-// testing. It reads the value from the ZOOKEEPER_PORT environment variable and
-// returns 2181 if it is not set.
-func GetZookeeperEnvPort() string {
-	port := os.Getenv("ZOOKEEPER_PORT")
-
-	if len(port) == 0 {
-		port = "2181"
-	}
-	return port
-}

From e1d587c36f083034dfa64bb59093add9be743668 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:17:03 +0100
Subject: [PATCH 56/88] Run memcached tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../memcached/stats/stats_integration_test.go | 44 ++++++-------------
 2 files changed, 16 insertions(+), 30 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e25330a02cdc..c8a5aed7e030 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -145,6 +145,8 @@ services:
 
   memcached:
     build: ./module/memcached/_meta
+    ports:
+      - 11211
 
   mongodb:
     build: ./module/mongodb/_meta
diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index 112033e08d6e..1672d600cb0e 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -20,46 +20,30 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "memcached")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestStats(t *testing.T) {
+	runner := compose.TestRunner{Service: "memcached"}
+
+	runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "memcached",
 		"metricsets": []string{"stats"},
-		"hosts":      []string{getEnvHost() + ":" + getEnvPort()},
-	}
-}
-
-func getEnvHost() string {
-	host := os.Getenv("MEMCACHED_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getEnvPort() string {
-	port := os.Getenv("MEMCACHED_PORT")
-
-	if len(port) == 0 {
-		port = "11211"
+		"hosts":      []string{host},
 	}
-	return port
 }

From 267a56180f371859cc94d3b95151d3edc105d8d1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 28 Oct 2018 17:34:09 +0100
Subject: [PATCH 57/88] Run uwsgi tests with testrunner

---
 .../uwsgi/status/status_integration_test.go   | 58 ++++++++++---------
 metricbeat/module/uwsgi/testing.go            | 38 ------------
 2 files changed, 31 insertions(+), 65 deletions(-)
 delete mode 100644 metricbeat/module/uwsgi/testing.go

diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 248b4612f478..b76380c4b95a 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -24,50 +24,54 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/uwsgi"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetchTCP(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "uwsgi_tcp")
+func testStatus(t *testing.T, service string) {
+	runner := compose.TestRunner{Service: service}
 
-	f := mbtest.NewEventsFetcher(t, getConfig("tcp"))
-	events, err := f.Fetch()
-	assert.NoError(t, err)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+			events, err := f.Fetch()
+			assert.NoError(t, err)
 
-	assert.True(t, len(events) > 0)
-	totals := findItems(events, "total")
-	assert.Equal(t, 1, len(totals))
+			assert.True(t, len(events) > 0)
+			totals := findItems(events, "total")
+			assert.Equal(t, 1, len(totals))
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func TestFetchHTTP(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "uwsgi_http")
-
-	f := mbtest.NewEventsFetcher(t, getConfig("http"))
-	events, err := f.Fetch()
-	assert.NoError(t, err)
+func TestStatusTCP(t *testing.T) {
+	testStatus(t, "uwsgi_tcp")
+}
 
-	assert.True(t, len(events) > 0)
-	totals := findItems(events, "total")
-	assert.Equal(t, 1, len(totals))
+func TestStatusHTTP(t *testing.T) {
+	testStatus(t, "uwsgi_http")
 }
 
-func getConfig(scheme string) map[string]interface{} {
+func getConfig(t *testing.T, service string, host string) map[string]interface{} {
 	conf := map[string]interface{}{
 		"module":     "uwsgi",
 		"metricsets": []string{"status"},
 	}
 
-	switch scheme {
-	case "tcp":
-		conf["hosts"] = []string{uwsgi.GetEnvTCPServer()}
-	case "http", "https":
-		conf["hosts"] = []string{uwsgi.GetEnvHTTPServer()}
+	switch service {
+	case "uwsgi_tcp":
+		conf["hosts"] = []string{"tcp://" + host}
+	case "uwsgi_http":
+		conf["hosts"] = []string{"http://" + host}
 	default:
-		conf["hosts"] = []string{uwsgi.GetEnvTCPServer()}
+		t.Errorf("Unexpected service: %s", service)
 	}
 	return conf
 }
diff --git a/metricbeat/module/uwsgi/testing.go b/metricbeat/module/uwsgi/testing.go
deleted file mode 100644
index b8c465354e6f..000000000000
--- a/metricbeat/module/uwsgi/testing.go
+++ /dev/null
@@ -1,38 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package uwsgi
-
-import "os"
-
-// GetEnvTCPServer returns uwsgi stat server host with tcp mode
-func GetEnvTCPServer() string {
-	env := os.Getenv("UWSGI_STAT_TCP_SERVER")
-	if len(env) == 0 {
-		env = "tcp://127.0.0.1:9191"
-	}
-	return env
-}
-
-// GetEnvHTTPServer returns uwsgi stat server host with http mode
-func GetEnvHTTPServer() string {
-	env := os.Getenv("UWSGI_STAT_HTTP_SERVER")
-	if len(env) == 0 {
-		env = "http://127.0.0.1:9192"
-	}
-	return env
-}

From 513976bc7902b605133349df9d8a77b809ea4a67 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 9 Nov 2018 22:19:33 +0100
Subject: [PATCH 58/88] Run traefik tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../traefik/health/health_integration_test.go | 69 +++++++++----------
 metricbeat/module/traefik/mtest/runner.go     | 29 ++++++++
 metricbeat/module/traefik/mtest/testing.go    | 26 +------
 4 files changed, 66 insertions(+), 60 deletions(-)
 create mode 100644 metricbeat/module/traefik/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index c8a5aed7e030..0fa903e38262 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -192,6 +192,8 @@ services:
 
   traefik:
     build: ./module/traefik/_meta
+    ports:
+     - 8080
 
   uwsgi_tcp:
     build: ./module/uwsgi/_meta
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index 8a49f755fa1f..c7fcabc4bad9 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -23,11 +23,11 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/traefik/mtest"
-
-	"github.com/stretchr/testify/assert"
 )
 
 func makeBadRequest(config map[string]interface{}) error {
@@ -41,38 +41,35 @@ func makeBadRequest(config map[string]interface{}) error {
 	return nil
 }
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "traefik")
-
-	config := mtest.GetConfig("health")
-
-	makeBadRequest(config)
-
-	ms := mbtest.NewReportingMetricSetV2(t, config)
-	reporter := &mbtest.CapturingReporterV2{}
-
-	ms.Fetch(reporter)
-	assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
-
-	event := reporter.GetEvents()[0]
-	assert.NotNil(t, event)
-	t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
-
-	responseCount, _ := event.MetricSetFields.GetValue("response.count")
-	assert.True(t, responseCount.(int64) >= 1)
-
-	badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
-	assert.True(t, badResponseCount.(float64) >= 1)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "traefik")
-
-	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health"))
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestHealth(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			config := mtest.GetConfig("health", r.Host())
+
+			makeBadRequest(config)
+
+			ms := mbtest.NewReportingMetricSetV2(t, config)
+			reporter := &mbtest.CapturingReporterV2{}
+
+			ms.Fetch(reporter)
+			assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
+
+			event := reporter.GetEvents()[0]
+			assert.NotNil(t, event)
+			t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
+
+			responseCount, _ := event.MetricSetFields.GetValue("response.count")
+			assert.True(t, responseCount.(int64) >= 1)
+
+			badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
+			assert.True(t, badResponseCount.(float64) >= 1)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/traefik/mtest/runner.go b/metricbeat/module/traefik/mtest/runner.go
new file mode 100644
index 000000000000..10ce0edf1623
--- /dev/null
+++ b/metricbeat/module/traefik/mtest/runner.go
@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "traefik",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/traefik/mtest/testing.go b/metricbeat/module/traefik/mtest/testing.go
index c71b9accb012..938f2ee7d4e3 100644
--- a/metricbeat/module/traefik/mtest/testing.go
+++ b/metricbeat/module/traefik/mtest/testing.go
@@ -17,33 +17,11 @@
 
 package mtest
 
-import "os"
-
-// GetEnvHost for Traefik
-func GetEnvHost() string {
-	host := os.Getenv("TRAEFIK_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-// GetEnvAPIPort for Traefik
-func GetEnvAPIPort() string {
-	port := os.Getenv("TRAEFIK_API_PORT")
-
-	if len(port) == 0 {
-		port = "8080"
-	}
-	return port
-}
-
 // GetConfig for Traefik
-func GetConfig(metricset string) map[string]interface{} {
+func GetConfig(metricset string, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "traefik",
 		"metricsets": []string{metricset},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvAPIPort()},
+		"hosts":      []string{host},
 	}
 }

From 8d3ed55185b524135c9e4e2355e41f36595981d2 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 10 Nov 2018 13:55:53 +0100
Subject: [PATCH 59/88] Run rabbitmq tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 ++
 .../connection/connection_integration_test.go | 23 ++++++++-------
 .../exchange/exchange_integration_test.go     | 29 ++++++++++---------
 .../module/rabbitmq/mtest/integration.go      | 14 ++-------
 metricbeat/module/rabbitmq/mtest/runner.go    | 29 +++++++++++++++++++
 .../rabbitmq/node/node_integration_test.go    | 23 ++++++++-------
 .../rabbitmq/queue/queue_integration_test.go  | 27 ++++++++---------
 7 files changed, 86 insertions(+), 61 deletions(-)
 create mode 100644 metricbeat/module/rabbitmq/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 0fa903e38262..69c66ee6f52b 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -172,6 +172,8 @@ services:
 
   rabbitmq:
     build: ./module/rabbitmq/_meta
+    ports:
+      - 15672
 
   redis:
     extends:
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index ae0f7a938685..420e65d5ccee 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -27,19 +27,20 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestConnection(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"connection"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index b8094935b8c1..da20d0e7c478 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -28,23 +28,24 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-		hasIn, _ := e.HasKey("messages.publish_in")
-		hasOut, _ := e.HasKey("messages.publish_out")
-		return hasIn && hasOut
+func TestExchange(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+				hasIn, _ := e.HasKey("messages.publish_in")
+				hasOut, _ := e.HasKey("messages.publish_out")
+				return hasIn && hasOut
+			})
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 	})
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"exchange"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/mtest/integration.go b/metricbeat/module/rabbitmq/mtest/integration.go
index 3160c0cf2598..cba6493391b3 100644
--- a/metricbeat/module/rabbitmq/mtest/integration.go
+++ b/metricbeat/module/rabbitmq/mtest/integration.go
@@ -18,35 +18,25 @@
 package mtest
 
 import (
-	"net"
 	"os"
 )
 
 // GetIntegrationConfig generates a base configuration with common values for
 // integration tests
-func GetIntegrationConfig() map[string]interface{} {
+func GetIntegrationConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":   "rabbitmq",
-		"hosts":    getTestRabbitMQHost(),
+		"hosts":    []string{host},
 		"username": getTestRabbitMQUsername(),
 		"password": getTestRabbitMQPassword(),
 	}
 }
 
 const (
-	rabbitmqDefaultHost     = "localhost"
-	rabbitmqDefaultPort     = "15672"
 	rabbitmqDefaultUsername = "guest"
 	rabbitmqDefaultPassword = "guest"
 )
 
-func getTestRabbitMQHost() string {
-	return net.JoinHostPort(
-		getenv("RABBITMQ_HOST", rabbitmqDefaultHost),
-		getenv("RABBITMQ_PORT", rabbitmqDefaultPort),
-	)
-}
-
 func getTestRabbitMQUsername() string {
 	return getenv("RABBITMQ_USERNAME", rabbitmqDefaultUsername)
 }
diff --git a/metricbeat/module/rabbitmq/mtest/runner.go b/metricbeat/module/rabbitmq/mtest/runner.go
new file mode 100644
index 000000000000..a70257d78b9b
--- /dev/null
+++ b/metricbeat/module/rabbitmq/mtest/runner.go
@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "rabbitmq",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index 938ac90024ec..ca0ae772b53a 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -27,19 +27,20 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	ms := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(ms, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestNode(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsReporterV2(ms, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"node"}
 	return config
 }
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index 9b348ac0a376..7f84b903fe49 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -28,22 +28,23 @@ import (
 	"github.com/elastic/beats/metricbeat/module/rabbitmq/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "rabbitmq")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-		hasTotal, _ := e.HasKey("messages.total")
-		return hasTotal
+func TestQueue(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+				hasTotal, _ := e.HasKey("messages.total")
+				return hasTotal
+			})
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
 	})
-	if err != nil {
-		t.Fatal("write", err)
-	}
 }
 
-func getConfig() map[string]interface{} {
-	config := mtest.GetIntegrationConfig()
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"queue"}
 	return config
 }

From 23d4e7fb81435cdd3800a6a62e3c043306c0d66c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 10 Nov 2018 14:37:45 +0100
Subject: [PATCH 60/88] Run postgresql tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../activity/activity_integration_test.go     |  72 ++++-----
 .../bgwriter/bgwriter_integration_test.go     |  79 ++++-----
 .../database/database_integration_test.go     |  80 ++++------
 .../{testing.go => mtest/runner.go}           |  23 ++-
 metricbeat/module/postgresql/mtest/testing.go |  49 ++++++
 .../statement/statement_integration_test.go   | 150 ++++++++----------
 7 files changed, 226 insertions(+), 229 deletions(-)
 rename metricbeat/module/postgresql/{testing.go => mtest/runner.go} (76%)
 create mode 100644 metricbeat/module/postgresql/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 69c66ee6f52b..59a4555679de 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -166,6 +166,8 @@ services:
 
   postgresql:
     build: ./module/postgresql/_meta
+    ports:
+      - 5432
 
   prometheus:
     build: ./module/prometheus/_meta
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index 0bc64f1de549..c48b399a8187 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -25,56 +25,44 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestActivity(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			assert.True(t, len(events) > 0)
+			event := events[0]
 
-	assert.True(t, len(events) > 0)
-	event := events[0]
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Check event fields
+			assert.Contains(t, event, "database")
+			db_oid := event["database"].(common.MapStr)["oid"].(int64)
+			assert.True(t, db_oid > 0)
 
-	// Check event fields
-	assert.Contains(t, event, "database")
-	db_oid := event["database"].(common.MapStr)["oid"].(int64)
-	assert.True(t, db_oid > 0)
+			assert.Contains(t, event, "pid")
+			assert.True(t, event["pid"].(int64) > 0)
 
-	assert.Contains(t, event, "pid")
-	assert.True(t, event["pid"].(int64) > 0)
+			assert.Contains(t, event, "user")
+			assert.Contains(t, event["user"].(common.MapStr), "name")
+			assert.Contains(t, event["user"].(common.MapStr), "id")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
 
-	assert.Contains(t, event, "user")
-	assert.Contains(t, event["user"].(common.MapStr), "name")
-	assert.Contains(t, event["user"].(common.MapStr), "id")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"activity"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 04203eee0c5e..3562d21d3f0a 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -25,59 +25,46 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestBgwriter(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			assert.Contains(t, event, "checkpoints")
+			assert.Contains(t, event, "buffers")
+			assert.Contains(t, event, "stats_reset")
 
-	assert.Contains(t, event, "checkpoints")
-	assert.Contains(t, event, "buffers")
-	assert.Contains(t, event, "stats_reset")
+			checkpoints := event["checkpoints"].(common.MapStr)
+			assert.Contains(t, checkpoints, "scheduled")
+			assert.Contains(t, checkpoints, "requested")
+			assert.Contains(t, checkpoints, "times")
 
-	checkpoints := event["checkpoints"].(common.MapStr)
-	assert.Contains(t, checkpoints, "scheduled")
-	assert.Contains(t, checkpoints, "requested")
-	assert.Contains(t, checkpoints, "times")
+			buffers := event["buffers"].(common.MapStr)
+			assert.Contains(t, buffers, "checkpoints")
+			assert.Contains(t, buffers, "clean")
+			assert.Contains(t, buffers, "clean_full")
+			assert.Contains(t, buffers, "backend")
+			assert.Contains(t, buffers, "backend_fsync")
+			assert.Contains(t, buffers, "allocated")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
 
-	buffers := event["buffers"].(common.MapStr)
-	assert.Contains(t, buffers, "checkpoints")
-	assert.Contains(t, buffers, "clean")
-	assert.Contains(t, buffers, "clean_full")
-	assert.Contains(t, buffers, "backend")
-	assert.Contains(t, buffers, "backend_fsync")
-	assert.Contains(t, buffers, "allocated")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"bgwriter"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index e09dd7077274..84b2ecfb3fbe 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -25,59 +25,45 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
+func TestDatabase(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
+			assert.True(t, len(events) > 0)
+			event := events[0]
 
-	assert.True(t, len(events) > 0)
-	event := events[0]
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+			// Check event fields
+			db_oid := event["oid"].(int64)
+			assert.True(t, db_oid > 0)
+			assert.Contains(t, event, "name")
+			_, ok := event["name"].(string)
+			assert.True(t, ok)
 
-	// Check event fields
-	db_oid := event["oid"].(int64)
-	assert.True(t, db_oid > 0)
-	assert.Contains(t, event, "name")
-	_, ok := event["name"].(string)
-	assert.True(t, ok)
-
-	rows := event["rows"].(common.MapStr)
-	assert.Contains(t, rows, "returned")
-	assert.Contains(t, rows, "fetched")
-	assert.Contains(t, rows, "inserted")
-	assert.Contains(t, rows, "updated")
-	assert.Contains(t, rows, "deleted")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"database"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+			rows := event["rows"].(common.MapStr)
+			assert.Contains(t, rows, "returned")
+			assert.Contains(t, rows, "fetched")
+			assert.Contains(t, rows, "inserted")
+			assert.Contains(t, rows, "updated")
+			assert.Contains(t, rows, "deleted")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/postgresql/testing.go b/metricbeat/module/postgresql/mtest/runner.go
similarity index 76%
rename from metricbeat/module/postgresql/testing.go
rename to metricbeat/module/postgresql/mtest/runner.go
index 13673c8c1d1a..af9ac5c0edfd 100644
--- a/metricbeat/module/postgresql/testing.go
+++ b/metricbeat/module/postgresql/mtest/runner.go
@@ -15,18 +15,15 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package postgresql
+package mtest
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-func GetEnvDSN() string {
-	return os.Getenv("POSTGRESQL_DSN")
-}
-
-func GetEnvUsername() string {
-	return os.Getenv("POSTGRESQL_USERNAME")
-}
-
-func GetEnvPassword() string {
-	return os.Getenv("POSTGRESQL_PASSWORD")
-}
+var (
+	Runner = compose.TestRunner{
+		Service:  "postgresql",
+		Parallel: true,
+	}
+)
diff --git a/metricbeat/module/postgresql/mtest/testing.go b/metricbeat/module/postgresql/mtest/testing.go
new file mode 100644
index 000000000000..348af73ed285
--- /dev/null
+++ b/metricbeat/module/postgresql/mtest/testing.go
@@ -0,0 +1,49 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"fmt"
+	"os"
+)
+
+const (
+	defaultUsername = "postgres"
+)
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	dsn := fmt.Sprintf("postgres://%s?sslmode=disable", host)
+	return map[string]interface{}{
+		"module":     "postgresql",
+		"metricsets": []string{metricset},
+		"hosts":      []string{dsn},
+		"username":   getEnvUsername(),
+		"password":   getEnvPassword(),
+	}
+}
+
+func getEnvUsername() string {
+	if username := os.Getenv("POSTGRESQL_USERNAME"); len(username) > 0 {
+		return username
+	}
+	return defaultUsername
+}
+
+func getEnvPassword() string {
+	return os.Getenv("POSTGRESQL_PASSWORD")
+}
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index b3449ed5c967..5055161c3173 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -25,89 +25,77 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/postgresql"
+	"github.com/elastic/beats/metricbeat/module/postgresql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	assert.True(t, len(events) > 0)
-	event := events[0]
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	assert.Contains(t, event, "user")
-	assert.Contains(t, event["user"].(common.MapStr), "id")
-
-	assert.Contains(t, event, "database")
-	db_oid := event["database"].(common.MapStr)["oid"].(int64)
-	assert.True(t, db_oid > 0)
-
-	assert.Contains(t, event, "query")
-	query := event["query"].(common.MapStr)
-	assert.Contains(t, query, "id")
-	assert.Contains(t, query, "text")
-	assert.Contains(t, query, "calls")
-	assert.Contains(t, query, "rows")
-
-	assert.Contains(t, query, "time")
-	time := query["time"].(common.MapStr)
-	assert.Contains(t, time, "total")
-	assert.Contains(t, time, "min")
-	assert.Contains(t, time, "max")
-	assert.Contains(t, time, "mean")
-	assert.Contains(t, time, "stddev")
-
-	assert.Contains(t, query["memory"], "shared")
-	memory := query["memory"].(common.MapStr)
-
-	assert.Contains(t, memory, "shared")
-	shared := memory["shared"].(common.MapStr)
-	assert.Contains(t, shared, "hit")
-	assert.Contains(t, shared, "read")
-	assert.Contains(t, shared, "dirtied")
-	assert.Contains(t, shared, "written")
-
-	assert.Contains(t, memory, "local")
-	local := memory["local"].(common.MapStr)
-	assert.Contains(t, local, "hit")
-	assert.Contains(t, local, "read")
-	assert.Contains(t, local, "dirtied")
-	assert.Contains(t, local, "written")
-
-	assert.Contains(t, memory, "temp")
-	temp := memory["temp"].(common.MapStr)
-	assert.Contains(t, temp, "read")
-	assert.Contains(t, temp, "written")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "postgresql")
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "postgresql",
-		"metricsets": []string{"statement"},
-		"hosts":      []string{postgresql.GetEnvDSN()},
-		"username":   postgresql.GetEnvUsername(),
-		"password":   postgresql.GetEnvPassword(),
-	}
+func TestStatement(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			assert.True(t, len(events) > 0)
+			event := events[0]
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			assert.Contains(t, event, "user")
+			assert.Contains(t, event["user"].(common.MapStr), "id")
+
+			assert.Contains(t, event, "database")
+			db_oid := event["database"].(common.MapStr)["oid"].(int64)
+			assert.True(t, db_oid > 0)
+
+			assert.Contains(t, event, "query")
+			query := event["query"].(common.MapStr)
+			assert.Contains(t, query, "id")
+			assert.Contains(t, query, "text")
+			assert.Contains(t, query, "calls")
+			assert.Contains(t, query, "rows")
+
+			assert.Contains(t, query, "time")
+			time := query["time"].(common.MapStr)
+			assert.Contains(t, time, "total")
+			assert.Contains(t, time, "min")
+			assert.Contains(t, time, "max")
+			assert.Contains(t, time, "mean")
+			assert.Contains(t, time, "stddev")
+
+			assert.Contains(t, query["memory"], "shared")
+			memory := query["memory"].(common.MapStr)
+
+			assert.Contains(t, memory, "shared")
+			shared := memory["shared"].(common.MapStr)
+			assert.Contains(t, shared, "hit")
+			assert.Contains(t, shared, "read")
+			assert.Contains(t, shared, "dirtied")
+			assert.Contains(t, shared, "written")
+
+			assert.Contains(t, memory, "local")
+			local := memory["local"].(common.MapStr)
+			assert.Contains(t, local, "hit")
+			assert.Contains(t, local, "read")
+			assert.Contains(t, local, "dirtied")
+			assert.Contains(t, local, "written")
+
+			assert.Contains(t, memory, "temp")
+			temp := memory["temp"].(common.MapStr)
+			assert.Contains(t, temp, "read")
+			assert.Contains(t, temp, "written")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From a337d77594f8364458677cf45e2b4f08850a35b5 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 11 Nov 2018 19:22:21 +0100
Subject: [PATCH 61/88] Set a default name for compose test scenarios without
 variables

---
 libbeat/tests/compose/runner.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 0ce16f3e1851..1123f29a5b64 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -121,6 +121,9 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		}
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
+		if desc == "" {
+			desc = "WithoutVars"
+		}
 
 		seq := make([]byte, 16)
 		rand.Read(seq)

From e4b209faff85f36f1141fcbc3bd9cc39bef31f39 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 11 Nov 2018 19:47:44 +0100
Subject: [PATCH 62/88] Run prometheus tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/prometheus/_meta/Dockerfile |  1 +
 .../collector/collector_integration_test.go   | 74 ++++++------------
 metricbeat/module/prometheus/mtest/testing.go | 37 +++++++++
 .../stats/stats_integration_test.go           | 78 ++++++-------------
 5 files changed, 90 insertions(+), 102 deletions(-)
 create mode 100644 metricbeat/module/prometheus/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 59a4555679de..4d7be27d0bba 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -171,6 +171,8 @@ services:
 
   prometheus:
     build: ./module/prometheus/_meta
+    ports:
+      - 9090
 
   rabbitmq:
     build: ./module/rabbitmq/_meta
diff --git a/metricbeat/module/prometheus/_meta/Dockerfile b/metricbeat/module/prometheus/_meta/Dockerfile
index 836c8777f629..f9d70de6ba5e 100644
--- a/metricbeat/module/prometheus/_meta/Dockerfile
+++ b/metricbeat/module/prometheus/_meta/Dockerfile
@@ -1,3 +1,4 @@
 FROM prom/prometheus:v1.5.1
+RUN mkdir /run # Needed to copy /run/compose_env on startup
 HEALTHCHECK --interval=1s --retries=90 CMD nc -w 1 localhost 9090 </dev/null
 EXPOSE 9090
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index d2f186e9d064..1669078e6437 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -20,11 +20,11 @@
 package collector
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/prometheus/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -32,54 +32,30 @@ import (
 // These tests are running with prometheus metrics as an example as this container is already available
 // Every prometheus exporter should work here.
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestCollector(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "prometheus",
-		"metricsets": []string{"collector"},
-		"hosts":      []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()},
-		"namespace":  "collector",
-	}
-}
-
-func getPrometheusEnvHost() string {
-	host := os.Getenv("PROMETHEUS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getPrometheusEnvPort() string {
-	port := os.Getenv("PROMETHEUS_PORT")
-
-	if len(port) == 0 {
-		port = "9090"
-	}
-	return port
+func getConfig(host string) map[string]interface{} {
+	config := mtest.GetConfig("collector", host)
+	config["namespace"] = "collector"
+	return config
 }
diff --git a/metricbeat/module/prometheus/mtest/testing.go b/metricbeat/module/prometheus/mtest/testing.go
new file mode 100644
index 000000000000..f6b7f18106f6
--- /dev/null
+++ b/metricbeat/module/prometheus/mtest/testing.go
@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	Runner = compose.TestRunner{
+		Service:  "prometheus",
+		Parallel: true,
+	}
+)
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "prometheus",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
+	}
+}
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 45e62466ec0e..0f18a5997530 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -20,67 +20,39 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/prometheus/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
+func TestStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check number of fields.
+			assert.Equal(t, 3, len(event))
+			assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check number of fields.
-	assert.Equal(t, 3, len(event))
-	assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "prometheus")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "prometheus",
-		"metricsets": []string{"stats"},
-		"hosts":      []string{getPrometheusEnvHost() + ":" + getPrometheusEnvPort()},
-	}
-}
-
-func getPrometheusEnvHost() string {
-	host := os.Getenv("PROMETHEUS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func getPrometheusEnvPort() string {
-	port := os.Getenv("PROMETHEUS_PORT")
-
-	if len(port) == 0 {
-		port = "9090"
-	}
-	return port
 }

From 45e195088087fa300a64cfa6975550716f03d4ad Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 12:06:18 +0100
Subject: [PATCH 63/88] Run mysql tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../module/mysql/{ => mtest}/testing.go       |  41 +++---
 .../module/mysql/mysql_integration_test.go    |  19 +--
 .../mysql/status/status_integration_test.go   | 125 ++++++++----------
 4 files changed, 92 insertions(+), 95 deletions(-)
 rename metricbeat/module/mysql/{ => mtest}/testing.go (57%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 4d7be27d0bba..2ff5657a3aed 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -157,6 +157,8 @@ services:
 
   mysql:
     build: ./module/mysql/_meta
+    ports:
+      - 3306
 
   nginx:
     build: ./module/nginx/_meta
diff --git a/metricbeat/module/mysql/testing.go b/metricbeat/module/mysql/mtest/testing.go
similarity index 57%
rename from metricbeat/module/mysql/testing.go
rename to metricbeat/module/mysql/mtest/testing.go
index 6036d717dc21..9a8410ccb8d9 100644
--- a/metricbeat/module/mysql/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -15,29 +15,36 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package mysql
+package mtest
 
 import (
-	"os"
-
 	"github.com/go-sql-driver/mysql"
+
+	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
-// Helper functions for testing used in the mysql MetricSets.
+var Runner = compose.TestRunner{
+	Service:  "mysql",
+	Parallel: true,
+}
 
-// GetMySQLEnvDSN returns the MySQL server DSN to use for testing. It
-// reads the value from the MYSQL_DSN environment variable and returns
-// root@tcp(127.0.0.1:3306)/ if it is not set.
-func GetMySQLEnvDSN() string {
-	dsn := os.Getenv("MYSQL_DSN")
+// GetDSN returns the MySQL server DSN to use for testing.
+func GetDSN(host string) string {
+	c := &mysql.Config{
+		Net:    "tcp",
+		Addr:   host,
+		User:   "root",
+		Passwd: "test",
+	}
+	return c.FormatDSN()
+}
 
-	if len(dsn) == 0 {
-		c := &mysql.Config{
-			Net:  "tcp",
-			Addr: "127.0.0.1:3306",
-			User: "root",
-		}
-		dsn = c.FormatDSN()
+// GetConfig returns the configuration for a mysql module
+func GetConfig(metricset, host string, raw bool) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "mysql",
+		"metricsets": []string{metricset},
+		"hosts":      []string{GetDSN(host)},
+		"raw":        raw,
 	}
-	return dsn
 }
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 32b90a519b62..602b617d215a 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -25,16 +25,17 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
-	_ "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/mysql/mtest"
 )
 
 func TestNewDB(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	db, err := NewDB(GetMySQLEnvDSN())
-	assert.NoError(t, err)
-
-	err = db.Ping()
-	assert.NoError(t, err)
+	mtest.Runner.Run(t, compose.Suite{
+		"NewDB": func(t *testing.T, r compose.R) {
+			db, err := NewDB(mtest.GetDSN(r.Host()))
+			assert.NoError(t, err)
+
+			err = db.Ping()
+			assert.NoError(t, err)
+		},
+	})
 }
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index e0a4898de35e..67f1c8742359 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -25,77 +25,64 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mysql"
+	"github.com/elastic/beats/metricbeat/module/mysql/mtest"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	f := mbtest.NewEventFetcher(t, getConfig(false))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	connections := event["connections"].(int64)
-	open := event["open"].(common.MapStr)
-	openTables := open["tables"].(int64)
-	openFiles := open["files"].(int64)
-	openStreams := open["streams"].(int64)
-
-	assert.True(t, connections > 0)
-	assert.True(t, openTables > 0)
-	assert.True(t, openFiles >= 0)
-	assert.True(t, openStreams == 0)
-}
-
-func TestFetchRaw(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mysql")
-
-	f := mbtest.NewEventFetcher(t, getConfig(true))
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
-	assert.True(t, cachedThreads >= 0)
-
-	rawData := event["raw"].(common.MapStr)
-
-	// Make sure field was removed from raw fields as in schema
-	_, exists := rawData["Threads_cached"]
-	assert.False(t, exists)
-
-	// Check a raw field if it is available
-	_, exists = rawData["Slow_launch_threads"]
-	assert.True(t, exists)
-}
-
-func TestData(t *testing.T) {
-	f := mbtest.NewEventFetcher(t, getConfig(false))
-
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig(raw bool) map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mysql",
-		"metricsets": []string{"status"},
-		"hosts":      []string{mysql.GetMySQLEnvDSN()},
-		"raw":        raw,
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			connections := event["connections"].(int64)
+			open := event["open"].(common.MapStr)
+			openTables := open["tables"].(int64)
+			openFiles := open["files"].(int64)
+			openStreams := open["streams"].(int64)
+
+			assert.True(t, connections > 0)
+			assert.True(t, openTables > 0)
+			assert.True(t, openFiles >= 0)
+			assert.True(t, openStreams == 0)
+		},
+		"FetchRaw": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
+			assert.True(t, cachedThreads >= 0)
+
+			rawData := event["raw"].(common.MapStr)
+
+			// Make sure field was removed from raw fields as in schema
+			_, exists := rawData["Threads_cached"]
+			assert.False(t, exists)
+
+			// Check a raw field if it is available
+			_, exists = rawData["Slow_launch_threads"]
+			assert.True(t, exists)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 972e304be58bd9ecb1f204e9220f0b166b6c04ed Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 16:50:40 +0100
Subject: [PATCH 64/88] Run php_fpm tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/php_fpm/mtest/testing.go    | 37 +++++++++++++++
 .../php_fpm/pool/pool_integration_test.go     | 46 +++++--------------
 .../process/process_integration_test.go       | 45 +++++-------------
 4 files changed, 61 insertions(+), 69 deletions(-)
 create mode 100644 metricbeat/module/php_fpm/mtest/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 2ff5657a3aed..9498c12ba49c 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -165,6 +165,8 @@ services:
 
   phpfpm:
     build: ./module/php_fpm/_meta
+    ports:
+      - 81
 
   postgresql:
     build: ./module/postgresql/_meta
diff --git a/metricbeat/module/php_fpm/mtest/testing.go b/metricbeat/module/php_fpm/mtest/testing.go
new file mode 100644
index 000000000000..7f427379fb49
--- /dev/null
+++ b/metricbeat/module/php_fpm/mtest/testing.go
@@ -0,0 +1,37 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var Runner = compose.TestRunner{
+	Service:  "phpfpm",
+	Parallel: true,
+}
+
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "php_fpm",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
+	}
+}
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index a2ff0d0f5c6e..332bb889a102 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -20,45 +20,21 @@
 package pool
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/php_fpm/mtest"
 )
 
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "phpfpm")
-	f := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "php_fpm",
-		"metricsets": []string{"pool"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("PHPFPM_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("PHPFPM_PORT")
-
-	if len(port) == 0 {
-		port = "81"
-	}
-	return port
+func TestPool(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go
index 182f4815de4f..66aa94e3acdd 100644
--- a/metricbeat/module/php_fpm/process/process_integration_test.go
+++ b/metricbeat/module/php_fpm/process/process_integration_test.go
@@ -20,44 +20,21 @@
 package process
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/php_fpm/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "phpfpm")
-	f := mbtest.NewReportingMetricSetV2(t, getConfig())
-	err := mbtest.WriteEventsReporterV2(f, t, "")
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "php_fpm",
-		"metricsets": []string{"process"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("PHPFPM_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("PHPFPM_PORT")
-
-	if len(port) == 0 {
-		port = "81"
-	}
-	return port
+func TestProcess(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, "")
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From ad8d2bc01316a5f139d6eb6d609735481b64930f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:03:01 +0100
Subject: [PATCH 65/88] Run nginx tests with testrunner

---
 metricbeat/docker-compose.yml                 |  2 +
 .../stubstatus/stubstatus_integration_test.go | 46 ++++++++++---------
 metricbeat/module/nginx/testing.go            | 35 --------------
 3 files changed, 27 insertions(+), 56 deletions(-)
 delete mode 100644 metricbeat/module/nginx/testing.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 9498c12ba49c..d9b5adfc920a 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -162,6 +162,8 @@ services:
 
   nginx:
     build: ./module/nginx/_meta
+    ports:
+      - 80
 
   phpfpm:
     build: ./module/php_fpm/_meta
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 779e400f02bc..7c2a441b97a8 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -24,40 +24,44 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/nginx"
 
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "nginx")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
+func TestStubstatus(t *testing.T) {
+	runner := compose.TestRunner{
+		Service:  "nginx",
+		Parallel: true,
 	}
 
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
 
-	// Check number of fields.
-	assert.Equal(t, 10, len(event))
-}
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-func TestData(t *testing.T) {
-	f := mbtest.NewEventFetcher(t, getConfig())
+			// Check number of fields.
+			assert.Equal(t, 10, len(event))
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nginx",
 		"metricsets": []string{"stubstatus"},
-		"hosts":      []string{nginx.GetNginxEnvHost()},
+		"hosts":      []string{host},
 	}
 }
diff --git a/metricbeat/module/nginx/testing.go b/metricbeat/module/nginx/testing.go
deleted file mode 100644
index cee26e1d11a5..000000000000
--- a/metricbeat/module/nginx/testing.go
+++ /dev/null
@@ -1,35 +0,0 @@
-// Licensed to Elasticsearch B.V. under one or more contributor
-// license agreements. See the NOTICE file distributed with
-// this work for additional information regarding copyright
-// ownership. Elasticsearch B.V. licenses this file to you under
-// the Apache License, Version 2.0 (the "License"); you may
-// not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package nginx
-
-/*
-Helper functions for testing used in the nginx metricsets
-*/
-
-import (
-	"os"
-)
-
-func GetNginxEnvHost() string {
-	host := os.Getenv("NGINX_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}

From 47fe92818923567249cc7834d0221f9a8d4ab757 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:26:41 +0100
Subject: [PATCH 66/88] Run mongodb tests with testrunner

---
 metricbeat/docker-compose.yml                 |   2 +
 .../collstats/collstats_integration_test.go   |  69 +++++-----
 .../dbstats/dbstats_integration_test.go       | 119 ++++++++----------
 .../metrics/metrics_intergration_test.go      |  65 ++++------
 .../module/mongodb/{ => mtest}/testing.go     |  35 ++----
 .../replstatus/replstatus_integration_test.go | 108 +++++++---------
 .../mongodb/status/status_integration_test.go |  71 +++++------
 7 files changed, 202 insertions(+), 267 deletions(-)
 rename metricbeat/module/mongodb/{ => mtest}/testing.go (54%)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index d9b5adfc920a..43476b466272 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -151,6 +151,8 @@ services:
   mongodb:
     build: ./module/mongodb/_meta
     command: mongod --replSet beats
+    ports:
+      - 27017
 
   munin:
     build: ./module/munin/_meta
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 4f860e98e8ed..98b5b0b5028f 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -26,46 +26,35 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	for _, event := range events {
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-		// Check a few event Fields
-		db := event["db"].(string)
-		assert.NotEqual(t, db, "")
-
-		collection := event["collection"].(string)
-		assert.NotEqual(t, collection, "")
-	}
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"collstats"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestCollstats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			for _, event := range events {
+				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+				// Check a few event Fields
+				db := event["db"].(string)
+				assert.NotEqual(t, db, "")
+
+				collection := event["collection"].(string)
+				assert.NotEqual(t, collection, "")
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 75fc73790ab9..8cea2158fe1e 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -26,71 +26,60 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	events, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	for _, event := range events {
-		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-		// Check a few event Fields
-		db := event["db"].(string)
-		assert.NotEqual(t, db, "")
-
-		collections := event["collections"].(int64)
-		assert.True(t, collections > 0)
-
-		objects := event["objects"].(int64)
-		assert.True(t, objects > 0)
-
-		avgObjSize, err := event.GetValue("avg_obj_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, avgObjSize.(int64) > 0)
-
-		dataSize, err := event.GetValue("data_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, dataSize.(int64) > 0)
-
-		storageSize, err := event.GetValue("storage_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, storageSize.(int64) > 0)
-
-		numExtents := event["num_extents"].(int64)
-		assert.True(t, numExtents >= 0)
-
-		indexes := event["indexes"].(int64)
-		assert.True(t, indexes >= 0)
-
-		indexSize, err := event.GetValue("index_size.bytes")
-		assert.NoError(t, err)
-		assert.True(t, indexSize.(int64) > 0)
-	}
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventsFetcher(t, getConfig())
-	err := mbtest.WriteEvents(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"dbstats"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestDBStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+			events, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			for _, event := range events {
+				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+				// Check a few event Fields
+				db := event["db"].(string)
+				assert.NotEqual(t, db, "")
+
+				collections := event["collections"].(int64)
+				assert.True(t, collections > 0)
+
+				objects := event["objects"].(int64)
+				assert.True(t, objects > 0)
+
+				avgObjSize, err := event.GetValue("avg_obj_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, avgObjSize.(int64) > 0)
+
+				dataSize, err := event.GetValue("data_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, dataSize.(int64) > 0)
+
+				storageSize, err := event.GetValue("storage_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, storageSize.(int64) > 0)
+
+				numExtents := event["num_extents"].(int64)
+				assert.True(t, numExtents >= 0)
+
+				indexes := event["indexes"].(int64)
+				assert.True(t, indexes >= 0)
+
+				indexSize, err := event.GetValue("index_size.bytes")
+				assert.NoError(t, err)
+				assert.True(t, indexSize.(int64) > 0)
+			}
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+			err := mbtest.WriteEvents(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index f39b950846f4..0d706716a403 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -26,44 +26,33 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	// Check a few event Fields
-	findCount, err := event.GetValue("commands.find.total")
-	assert.NoError(t, err)
-	assert.True(t, findCount.(int64) >= 0)
-
-	deletedDocuments, err := event.GetValue("document.deleted")
-	assert.NoError(t, err)
-	assert.True(t, deletedDocuments.(int64) >= 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"metrics"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestMetrics(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			// Check a few event Fields
+			findCount, err := event.GetValue("commands.find.total")
+			assert.NoError(t, err)
+			assert.True(t, findCount.(int64) >= 0)
+
+			deletedDocuments, err := event.GetValue("document.deleted")
+			assert.NoError(t, err)
+			assert.True(t, deletedDocuments.(int64) >= 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
diff --git a/metricbeat/module/mongodb/testing.go b/metricbeat/module/mongodb/mtest/testing.go
similarity index 54%
rename from metricbeat/module/mongodb/testing.go
rename to metricbeat/module/mongodb/mtest/testing.go
index 31d380b8322c..5d4200825a81 100644
--- a/metricbeat/module/mongodb/testing.go
+++ b/metricbeat/module/mongodb/mtest/testing.go
@@ -15,32 +15,21 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package mongodb
+package mtest
 
-import "os"
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
 
-// Helper functions for testing mongodb metricsets.
-
-// GetEnvHost returns the hostname of the Mongodb server to use for testing.
-// It reads the value from the MONGODB_HOST environment variable and returns
-// 127.0.0.1 if it is not set.
-func GetEnvHost() string {
-	host := os.Getenv("MONGODB_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
+var Runner = compose.TestRunner{
+	Service:  "mongodb",
+	Parallel: true,
 }
 
-// GetEnvPort returns the port of the Mongodb server to use for testing.
-// It reads the value from the MONGODB_PORT environment variable and returns
-// 27017 if it is not set.
-func GetEnvPort() string {
-	port := os.Getenv("MONGODB_PORT")
-
-	if len(port) == 0 {
-		port = "27017"
+func GetConfig(metricset, host string) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "mongodb",
+		"metricsets": []string{metricset},
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index 526a81aba8fb..c1a9b721b4d5 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -31,70 +31,58 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	err := initiateReplicaSet(t)
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	oplog := event["oplog"].(common.MapStr)
-	allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
-	assert.True(t, allocated >= 0)
-
-	used := oplog["size"].(common.MapStr)["used"].(float64)
-	assert.True(t, used > 0)
-
-	firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
-	assert.True(t, firstTs >= 0)
-
-	window := oplog["window"].(int64)
-	assert.True(t, window >= 0)
-
-	members := event["members"].(common.MapStr)
-	primary := members["primary"].(common.MapStr)
-	assert.NotEmpty(t, primary["host"].(string))
-	assert.True(t, primary["optime"].(int64) > 0)
-
-	set := event["set_name"].(string)
-	assert.Equal(t, set, "beats")
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
+func TestReplStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			err := initiateReplicaSet(t, r.Host())
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			oplog := event["oplog"].(common.MapStr)
+			allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
+			assert.True(t, allocated >= 0)
+
+			used := oplog["size"].(common.MapStr)["used"].(float64)
+			assert.True(t, used > 0)
+
+			firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
+			assert.True(t, firstTs >= 0)
+
+			window := oplog["window"].(int64)
+			assert.True(t, window >= 0)
+
+			members := event["members"].(common.MapStr)
+			primary := members["primary"].(common.MapStr)
+			assert.NotEmpty(t, primary["host"].(string))
+			assert.True(t, primary["optime"].(int64) > 0)
+
+			set := event["set_name"].(string)
+			assert.Equal(t, set, "beats")
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }
 
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"replstatus"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
-}
-
-func initiateReplicaSet(t *testing.T) error {
-	url := getConfig()["hosts"].([]string)[0]
-
+func initiateReplicaSet(t *testing.T, url string) error {
 	dialInfo, err := mgo.ParseURL(url)
 	if err != nil {
 		return err
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 2140b0c181be..91eab6daf6ce 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -27,47 +27,36 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/mongodb"
+	"github.com/elastic/beats/metricbeat/module/mongodb/mtest"
 )
 
-func TestFetch(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	event, err := f.Fetch()
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-
-	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-	// Check event fields
-	current := event["connections"].(common.MapStr)["current"].(int64)
-	assert.True(t, current >= 0)
-
-	available := event["connections"].(common.MapStr)["available"].(int64)
-	assert.True(t, available > 0)
-
-	pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
-	assert.True(t, pageFaults >= 0)
-}
-
-func TestData(t *testing.T) {
-	t.Skip("ignoring tests with EnsureUp by now")
-	compose.EnsureUp(t, "mongodb")
-
-	f := mbtest.NewEventFetcher(t, getConfig())
-	err := mbtest.WriteEvent(f, t)
-	if err != nil {
-		t.Fatal("write", err)
-	}
-}
-
-func getConfig() map[string]interface{} {
-	return map[string]interface{}{
-		"module":     "mongodb",
-		"metricsets": []string{"status"},
-		"hosts":      []string{mongodb.GetEnvHost() + ":" + mongodb.GetEnvPort()},
-	}
+func TestStatus(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			event, err := f.Fetch()
+			if !assert.NoError(t, err) {
+				t.FailNow()
+			}
+
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+			// Check event fields
+			current := event["connections"].(common.MapStr)["current"].(int64)
+			assert.True(t, current >= 0)
+
+			available := event["connections"].(common.MapStr)["available"].(int64)
+			assert.True(t, available > 0)
+
+			pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
+			assert.True(t, pageFaults >= 0)
+		},
+		"Data": func(t *testing.T, r compose.R) {
+			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+			err := mbtest.WriteEvent(f, t)
+			if err != nil {
+				t.Fatal("write", err)
+			}
+		},
+	})
 }

From 19737034d3065f7ba88d78f081e767ca6c768368 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 17:35:08 +0100
Subject: [PATCH 67/88] Remove unused EnsureUp

---
 libbeat/tests/compose/compose.go | 43 --------------------------------
 1 file changed, 43 deletions(-)

diff --git a/libbeat/tests/compose/compose.go b/libbeat/tests/compose/compose.go
index de1b19ebd8f8..0afb9a558333 100644
--- a/libbeat/tests/compose/compose.go
+++ b/libbeat/tests/compose/compose.go
@@ -21,51 +21,8 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
-	"strconv"
-	"testing"
 )
 
-// EnsureUp starts all the requested services (must be defined in docker-compose.yml)
-// with a default timeout of 300 seconds
-func EnsureUp(t *testing.T, services ...string) {
-	EnsureUpWithTimeout(t, 300, services...)
-}
-
-// EnsureUpWithTimeout starts all the requested services (must be defined in docker-compose.yml)
-// Wait for `timeout` seconds for health
-func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
-	// The NO_COMPOSE env variables makes it possible to skip the starting of the environment.
-	// This is useful if the service is already running locally.
-	if noCompose, err := strconv.ParseBool(os.Getenv("NO_COMPOSE")); err == nil && noCompose {
-		return
-	}
-
-	name := os.Getenv("DOCKER_COMPOSE_PROJECT_NAME")
-	compose, err := getComposeProject(name)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Kill no longer used containers
-	err = compose.KillOld(services)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	for _, service := range services {
-		err = compose.Start(service)
-		if err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	// Wait for health
-	err = compose.Wait(timeout, services...)
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func findComposePath() (string, error) {
 	// find docker-compose
 	path, err := os.Getwd()

From 7a4a29d0c29d181a0a62e70ef0757e87b3dbebaf Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 18:11:14 +0100
Subject: [PATCH 68/88] Fix default description

---
 libbeat/tests/compose/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 1123f29a5b64..51794fdd468f 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -122,7 +122,7 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 		sort.Strings(vars)
 		desc := strings.Join(vars, ",")
 		if desc == "" {
-			desc = "WithoutVars"
+			desc = "WithoutOptions"
 		}
 
 		seq := make([]byte, 16)

From 78a9fe18f24ec0b81ba954e048bb18d8d221ed9a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 18:46:55 +0100
Subject: [PATCH 69/88] Allow to use runner ctl as T

---
 libbeat/tests/compose/runner.go | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 51794fdd468f..d9746c62dc0e 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -41,7 +41,7 @@ type TestRunner struct {
 	Timeout  int
 }
 
-type Suite map[string]func(t *testing.T, r R)
+type Suite map[string]interface{}
 
 type RunnerOptions map[string][]string
 
@@ -76,7 +76,16 @@ func (r *TestRunner) scenarios() []map[string]string {
 
 func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
 	for name, test := range tests {
-		t.Run(name, func(t *testing.T) { test(t, ctl) })
+		var testFunc func(t *testing.T)
+		switch f := test.(type) {
+		case func(R):
+			testFunc = func(t *testing.T) { f(ctl) }
+		case func(*testing.T, R):
+			testFunc = func(t *testing.T) { f(t, ctl) }
+		default:
+			t.Fatalf("incorrect test suite function '%s'", name)
+		}
+		t.Run(name, testFunc)
 	}
 }
 
@@ -90,8 +99,8 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	t.Logf("Test host overriden by %s=%s", env, host)
 
 	ctl := &runnerControl{
+		T:    t,
 		host: host,
-		t:    t,
 	}
 	r.runSuite(t, tests, ctl)
 	return true
@@ -159,8 +168,8 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			ctl := &runnerControl{
+				T:        t,
 				host:     host,
-				t:        t,
 				scenario: s,
 			}
 			r.runSuite(t, tests, ctl)
@@ -169,7 +178,10 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	}
 }
 
+// R extends the testing.T interface with methods that expose information about current scenario
 type R interface {
+	testing.TB
+
 	Host() string
 	Hostname() string
 	Port() string
@@ -178,7 +190,8 @@ type R interface {
 }
 
 type runnerControl struct {
-	t        *testing.T
+	*testing.T
+
 	host     string
 	scenario map[string]string
 }

From 605e642a468c7cf64704a619515c732c41d6f1ef Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Mon, 12 Nov 2018 20:04:00 +0100
Subject: [PATCH 70/88] Get compose hosts from docker in python tests

---
 libbeat/tests/system/beat/compose.py          | 28 +++++++++++++------
 metricbeat/docker-compose.yml                 | 18 ++++++++++++
 .../elasticsearch/test_elasticsearch.py       |  3 +-
 metricbeat/tests/system/test_aerospike.py     |  3 +-
 metricbeat/tests/system/test_apache.py        |  7 +----
 metricbeat/tests/system/test_base.py          | 10 ++-----
 metricbeat/tests/system/test_ceph.py          |  3 +-
 metricbeat/tests/system/test_config.py        |  2 +-
 metricbeat/tests/system/test_couchbase.py     |  3 +-
 metricbeat/tests/system/test_dropwizard.py    |  3 +-
 metricbeat/tests/system/test_envoyproxy.py    |  3 +-
 metricbeat/tests/system/test_haproxy.py       |  8 +++---
 metricbeat/tests/system/test_http.py          |  3 +-
 metricbeat/tests/system/test_jolokia.py       |  3 +-
 metricbeat/tests/system/test_kafka.py         |  3 +-
 metricbeat/tests/system/test_kibana.py        |  3 +-
 metricbeat/tests/system/test_kubernetes.py    | 12 ++------
 metricbeat/tests/system/test_logstash.py      |  3 +-
 metricbeat/tests/system/test_memcached.py     |  3 +-
 metricbeat/tests/system/test_mongodb.py       |  3 +-
 metricbeat/tests/system/test_munin.py         |  3 +-
 metricbeat/tests/system/test_mysql.py         |  4 +--
 metricbeat/tests/system/test_phpfpm.py        |  3 +-
 metricbeat/tests/system/test_postgresql.py    |  9 +++---
 metricbeat/tests/system/test_prometheus.py    |  3 +-
 metricbeat/tests/system/test_redis.py         |  8 +++---
 metricbeat/tests/system/test_uwsgi.py         |  6 ++--
 metricbeat/tests/system/test_zookeeper.py     |  3 +-
 28 files changed, 76 insertions(+), 87 deletions(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index 54387f006df7..c322c4e34ce5 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -89,21 +89,33 @@ def compose_down(cls):
         """
         Stop all running containers
         """
+        if os.environ.get('NO_COMPOSE'):
+            return
+
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
     @classmethod
-    def compose_hosts(cls):
+    def compose_host(cls, service=None, port=None):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
             return []
 
-        hosts = []
-        for container in cls.compose_project().containers(service_names=cls.COMPOSE_SERVICES):
-            network_settings = container.inspect()['NetworkSettings']
-            for network in network_settings['Networks'].values():
-                if network['IPAddress']:
-                    hosts.append(network['IPAddress'])
-        return hosts
+        if service is None:
+            service = cls.COMPOSE_SERVICES[0]
+
+        host_env = os.environ.get(service.upper() + "_HOST")
+        if host_env:
+            return host_env
+
+        container = cls.compose_project().containers(service_names=[service])[0]
+        info = container.inspect()
+
+        if port is None:
+            portsConfig = info['HostConfig']['PortBindings']
+            port = portsConfig.keys()[0]
+
+        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        return "localhost:%s" % hostPort
 
     @classmethod
     def compose_project(cls):
diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 43476b466272..6bfcafb656bf 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -34,9 +34,13 @@ services:
 
   ceph:
     build: ./module/ceph/_meta
+    ports:
+      - 5000
 
   couchbase:
     build: ./module/couchbase/_meta
+    ports:
+      - 8091
 
   dropwizard:
     build: ./module/dropwizard/_meta
@@ -60,16 +64,28 @@ services:
 
   haproxy:
     build: ./module/haproxy/_meta
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   haproxy_1_6:
     build:
       context: ./module/haproxy/_meta
       dockerfile: Dockerfile.1.6
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   haproxy_1_7:
     build:
       context: ./module/haproxy/_meta
       dockerfile: Dockerfile.1.7
+    ports:
+      - 14567
+      - 14568
+      - 14569
 
   http:
     build: ./module/http/_meta
@@ -156,6 +172,8 @@ services:
 
   munin:
     build: ./module/munin/_meta
+    ports:
+      - 4949
 
   mysql:
     build: ./module/mysql/_meta
diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py
index fbc24822c518..308f77d1c7a9 100644
--- a/metricbeat/module/elasticsearch/test_elasticsearch.py
+++ b/metricbeat/module/elasticsearch/test_elasticsearch.py
@@ -45,8 +45,7 @@ def test_metricsets(self, metricset):
                              ["service.name"], extras={"index_recovery.active_only": "false"})
 
     def get_hosts(self):
-        return [os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ES_PORT', '9200')]
+        return [self.compose_host()]
 
     def create_ml_job(self):
         es = Elasticsearch(self.get_hosts())
diff --git a/metricbeat/tests/system/test_aerospike.py b/metricbeat/tests/system/test_aerospike.py
index 9d44794603a2..6f118fda0d8c 100644
--- a/metricbeat/tests/system/test_aerospike.py
+++ b/metricbeat/tests/system/test_aerospike.py
@@ -16,5 +16,4 @@ def test_namespace(self):
         self.check_metricset("aerospike", "namespace", self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('AEROSPIKE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('AEROSPIKE_PORT', '3000')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_apache.py b/metricbeat/tests/system/test_apache.py
index a74f7b980896..41fa7b313003 100644
--- a/metricbeat/tests/system/test_apache.py
+++ b/metricbeat/tests/system/test_apache.py
@@ -78,8 +78,7 @@ def verify_fields(self, evt):
         # There are more fields that could be checked.
 
     def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('APACHE_PORT', '80')]
+        return ['http://' + self.compose_host()]
 
 
 class ApacheOldStatusTest(ApacheStatusTest):
@@ -91,7 +90,3 @@ def verify_fields(self, evt):
         apache_status = evt["apache"]["status"]
         self.assertItemsEqual(
             self.de_dot(APACHE_OLD_STATUS_FIELDS), apache_status.keys())
-
-    def get_hosts(self):
-        return ['http://' + os.getenv('APACHE_OLD_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('APACHE_PORT', '80')]
diff --git a/metricbeat/tests/system/test_base.py b/metricbeat/tests/system/test_base.py
index 770ea0ef7160..cb50d0cb835d 100644
--- a/metricbeat/tests/system/test_base.py
+++ b/metricbeat/tests/system/test_base.py
@@ -78,16 +78,10 @@ def test_dashboards(self):
         assert self.log_contains("Kibana dashboards successfully loaded.")
 
     def get_elasticsearch_url(self):
-        return "http://{host}:{port}".format(
-            host=os.getenv("ES_HOST", self.compose_hosts()[1]),
-            port=os.getenv("ES_PORT", "9200"),
-        )
+        return "http://" + self.compose_host("elasticsearch")
 
     def get_kibana_url(self):
         """
         Returns kibana host URL
         """
-        return "http://{host}:{port}".format(
-            host=os.getenv("KIBANA_HOST", self.compose_hosts()[0]),
-            port=os.getenv("KIBANA_PORT", "5601"),
-        )
+        return "http://" + self.compose_host("kibana")
diff --git a/metricbeat/tests/system/test_ceph.py b/metricbeat/tests/system/test_ceph.py
index fb90acf8a283..0cd10b29a229 100644
--- a/metricbeat/tests/system/test_ceph.py
+++ b/metricbeat/tests/system/test_ceph.py
@@ -24,5 +24,4 @@ def test_ceph(self, metricset):
         self.check_metricset("ceph", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('CEPH_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('CEPH_PORT', '5000')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_config.py b/metricbeat/tests/system/test_config.py
index 0804e5e0bee9..d4ac45a9da45 100644
--- a/metricbeat/tests/system/test_config.py
+++ b/metricbeat/tests/system/test_config.py
@@ -27,4 +27,4 @@ def test_invalid_config_with_removed_settings(self):
                                  " has been removed")
 
     def get_host(self):
-        return 'http://' + os.getenv('ES_HOST', self.compose_hosts()[0]) + ':' + os.getenv('ES_PORT', '9200')
+        return 'http://' + self.compose_host()
diff --git a/metricbeat/tests/system/test_couchbase.py b/metricbeat/tests/system/test_couchbase.py
index 49d11b3f21ba..f8b2f619e09a 100644
--- a/metricbeat/tests/system/test_couchbase.py
+++ b/metricbeat/tests/system/test_couchbase.py
@@ -22,5 +22,4 @@ def test_couchbase(self, metricset):
         self.check_metricset("couchbase", metricset, self.get_hosts(), self.FIELDS)
 
     def get_hosts(self):
-        return [os.getenv('COUCHBASE_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('COUCHBASE_PORT', '8091')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_dropwizard.py b/metricbeat/tests/system/test_dropwizard.py
index 651fb4968ec3..34297426f2d5 100644
--- a/metricbeat/tests/system/test_dropwizard.py
+++ b/metricbeat/tests/system/test_dropwizard.py
@@ -30,5 +30,4 @@ def test_dropwizard(self):
         self.assertTrue(len(output) >= 1)
 
     def get_hosts(self):
-        return [os.getenv('DROPWIZARD_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('DROPWIZARD_PORT', '8080')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py
index b30bcac1e208..013cbb13aa4c 100644
--- a/metricbeat/tests/system/test_envoyproxy.py
+++ b/metricbeat/tests/system/test_envoyproxy.py
@@ -31,5 +31,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ENVOYPROXY_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ENVOYPROXY_PORT', '9901')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_haproxy.py b/metricbeat/tests/system/test_haproxy.py
index 8b7dd6e976fe..9a3c0f192f84 100644
--- a/metricbeat/tests/system/test_haproxy.py
+++ b/metricbeat/tests/system/test_haproxy.py
@@ -32,7 +32,7 @@ def test_info_socket(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["info"],
-            "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)],
+            "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))],
             "period": "5s"
         }])
         self._test_info()
@@ -59,7 +59,7 @@ def test_stat_socket(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["stat"],
-            "hosts": ["tcp://%s:%d" % (self.compose_hosts()[0], 14567)],
+            "hosts": ["tcp://%s" % (self.compose_host(port="14567/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
@@ -72,7 +72,7 @@ def test_stat_http(self):
         self.render_config_template(modules=[{
             "name": "haproxy",
             "metricsets": ["stat"],
-            "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14568)],
+            "hosts": ["http://%s/stats" % (self.compose_host(port="14568/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
@@ -87,7 +87,7 @@ def test_stat_http_auth(self):
             "metricsets": ["stat"],
             "username": "admin",
             "password": "admin",
-            "hosts": ["http://%s:%d/stats" % (self.compose_hosts()[0], 14569)],
+            "hosts": ["http://%s/stats" % (self.compose_host(port="14569/tcp"))],
             "period": "5s"
         }])
         self._test_stat()
diff --git a/metricbeat/tests/system/test_http.py b/metricbeat/tests/system/test_http.py
index 64e6239be92f..c4b343572a2b 100644
--- a/metricbeat/tests/system/test_http.py
+++ b/metricbeat/tests/system/test_http.py
@@ -76,5 +76,4 @@ def test_server(self):
         self.assert_fields_are_documented(evt)
 
     def get_host(self):
-        host = self.compose_hosts()[0]
-        return "http://" + os.getenv('HTTP_HOST', host) + ':' + os.getenv('HTTP_PORT', '8080')
+        return "http://" + self.compose_host()
diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py
index 3ca676999566..b44f03985b69 100644
--- a/metricbeat/tests/system/test_jolokia.py
+++ b/metricbeat/tests/system/test_jolokia.py
@@ -50,5 +50,4 @@ def test_jmx(self, mbean):
         assert evt["jolokia"]["test"]["gc"]["collection_count"] >= 0
 
     def get_hosts(self):
-        return [os.getenv('JOLOKIA_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('JOLOKIA_PORT', '8778')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_kafka.py b/metricbeat/tests/system/test_kafka.py
index a7ba2d26e288..4f030515deac 100644
--- a/metricbeat/tests/system/test_kafka.py
+++ b/metricbeat/tests/system/test_kafka.py
@@ -43,8 +43,7 @@ def create_topic(self):
         producer.send('foobar', b'some_message_bytes')
 
     def get_hosts(self):
-        return [self.compose_hosts()[0] + ':' +
-                os.getenv('KAFKA_PORT', '9092')]
+        return [self.compose_host()]
 
 
 class Kafka_1_1_0_Test(KafkaTest):
diff --git a/metricbeat/tests/system/test_kibana.py b/metricbeat/tests/system/test_kibana.py
index 456588201ae2..33f6f376c1be 100644
--- a/metricbeat/tests/system/test_kibana.py
+++ b/metricbeat/tests/system/test_kibana.py
@@ -48,8 +48,7 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('KIBANA_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('KIBANA_PORT', '5601')]
+        return [self.compose_host("kibana")]
 
     def get_version(self):
         host = self.get_hosts()[0]
diff --git a/metricbeat/tests/system/test_kubernetes.py b/metricbeat/tests/system/test_kubernetes.py
index abb14161474c..5c469bc3902e 100644
--- a/metricbeat/tests/system/test_kubernetes.py
+++ b/metricbeat/tests/system/test_kubernetes.py
@@ -77,16 +77,8 @@ def _test_metricset(self, metricset, expected_events, hosts):
 
     @classmethod
     def get_kubelet_hosts(cls):
-        return [
-            "http://" +
-            os.getenv('KUBELET_HOST', self.compose_hosts()[0]) + ':' +
-            os.getenv('KUBELET_PORT', '10255')
-        ]
+        return [self.compose_host("kubernetes")]
 
     @classmethod
     def get_kube_state_hosts(cls):
-        return [
-            "http://" +
-            os.getenv('KUBE_STATE_METRICS_HOST', self.compose_hosts()[1]) + ':' +
-            os.getenv('KUBE_STATE_METRICS_PORT', '18080')
-        ]
+        return [self.compose_host("kubestate")]
diff --git a/metricbeat/tests/system/test_logstash.py b/metricbeat/tests/system/test_logstash.py
index 816aaf11cd7e..2f6c5b2c38db 100644
--- a/metricbeat/tests/system/test_logstash.py
+++ b/metricbeat/tests/system/test_logstash.py
@@ -55,5 +55,4 @@ def test_node_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('LOGSTASH_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('LOGSTASH_PORT', '9600')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_memcached.py b/metricbeat/tests/system/test_memcached.py
index c41d34b7cc66..38c325a29f06 100644
--- a/metricbeat/tests/system/test_memcached.py
+++ b/metricbeat/tests/system/test_memcached.py
@@ -31,5 +31,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MEMCACHED_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MEMCACHED_PORT', '11211')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_mongodb.py b/metricbeat/tests/system/test_mongodb.py
index 88327e80a23b..257721b3bc5e 100644
--- a/metricbeat/tests/system/test_mongodb.py
+++ b/metricbeat/tests/system/test_mongodb.py
@@ -36,5 +36,4 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MONGODB_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MONGODB_PORT', '27017')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_munin.py b/metricbeat/tests/system/test_munin.py
index 391d8f845d5d..049dc9c63fd6 100644
--- a/metricbeat/tests/system/test_munin.py
+++ b/metricbeat/tests/system/test_munin.py
@@ -34,5 +34,4 @@ def test_munin_node(self):
         assert evt["munin"][namespace]["cpu"]["user"] > 0
 
     def get_hosts(self):
-        return [os.getenv('MUNIN_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('MUNIN_PORT', '4949')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_mysql.py b/metricbeat/tests/system/test_mysql.py
index 4087b141e49f..244d5aefa7b2 100644
--- a/metricbeat/tests/system/test_mysql.py
+++ b/metricbeat/tests/system/test_mysql.py
@@ -40,6 +40,4 @@ def test_status(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        host = os.getenv('MYSQL_HOST', self.compose_hosts()[0])
-        port = os.getenv('MYSQL_PORT', '3306')
-        return [os.getenv('MYSQL_DSN', 'root:test@tcp({}:{})/'.format(host, port))]
+        return ['root:test@tcp({})/'.format(self.compose_host())]
diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py
index 4049f96748a1..df20358e9d76 100644
--- a/metricbeat/tests/system/test_phpfpm.py
+++ b/metricbeat/tests/system/test_phpfpm.py
@@ -34,5 +34,4 @@ def test_info(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('PHPFPM_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('PHPFPM_PORT', '81')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_postgresql.py b/metricbeat/tests/system/test_postgresql.py
index 6f2e11937458..43a8db4a3858 100644
--- a/metricbeat/tests/system/test_postgresql.py
+++ b/metricbeat/tests/system/test_postgresql.py
@@ -20,12 +20,11 @@ def common_checks(self, output):
 
     def get_hosts(self):
         username = "postgres"
-        host = self.compose_hosts()[0]
-        port = 5432
-        dsn = "postgres://{}:{}?sslmode=disable".format(host, port)
+        host = self.compose_host()
+        dsn = "postgres://{}?sslmode=disable".format(host)
         return (
-            [os.getenv("POSTGRESQL_DSN", dsn)],
-            os.getenv("POSTGRESQL_USERNAME", username),
+            [dsn],
+            username,
             os.getenv("POSTGRESQL_PASSWORD"),
         )
 
diff --git a/metricbeat/tests/system/test_prometheus.py b/metricbeat/tests/system/test_prometheus.py
index ccf6bbf220fa..dc94b4b91449 100644
--- a/metricbeat/tests/system/test_prometheus.py
+++ b/metricbeat/tests/system/test_prometheus.py
@@ -34,5 +34,4 @@ def test_stats(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["http://" + os.getenv('PROMETHEUS_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('PROMETHEUS_PORT', '9090')]
+        return [self.compose_host()]
diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index a1404390860a..728ab5b58a6a 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -59,9 +59,10 @@ def test_keyspace(self):
         """
 
         # At least one event must be inserted so db stats exist
+        host, port = self.compose_host().split(":")
         r = redis.StrictRedis(
-            host=os.getenv('REDIS_HOST', self.compose_hosts()[0]),
-            port=os.getenv('REDIS_PORT', '6379'),
+            host=host,
+            port=port,
             db=0)
         r.set('foo', 'bar')
 
@@ -120,8 +121,7 @@ def test_module_processors(self):
         self.assertItemsEqual(self.de_dot(CPU_FIELDS), redis_info["cpu"].keys())
 
     def get_hosts(self):
-        return [os.getenv('REDIS_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('REDIS_PORT', '6379')]
+        return [self.compose_host()]
 
 
 class TestRedis4(Test):
diff --git a/metricbeat/tests/system/test_uwsgi.py b/metricbeat/tests/system/test_uwsgi.py
index 7c4bfd79ddea..a35576fb85e4 100644
--- a/metricbeat/tests/system/test_uwsgi.py
+++ b/metricbeat/tests/system/test_uwsgi.py
@@ -68,13 +68,11 @@ def test_status(self):
         self.common_checks(output)
 
     def get_host(self):
-        return os.getenv("UWSGI_STAT_TCP_SERVER",
-                         "tcp://{}:9191".format(self.compose_hosts()[0]))
+        return "tcp://" + self.compose_host()
 
 
 class TestHTTP(Test):
     COMPOSE_SERVICES = ['uwsgi_http']
 
     def get_host(self):
-        return os.getenv("UWSGI_STAT_HTTP_SERVER",
-                         "http://{}:9191".format(self.compose_hosts()[0]))
+        return "http://" + self.compose_host()
diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py
index 6d35f9d4d4cd..23260a3eb2d7 100644
--- a/metricbeat/tests/system/test_zookeeper.py
+++ b/metricbeat/tests/system/test_zookeeper.py
@@ -51,5 +51,4 @@ def test_output(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('ZOOKEEPER_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('ZOOKEEPER_PORT', '2181')]
+        return [self.compose_host()]

From 3fc2715c3f09225eb859ad64a300b7bed6c585df Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 14 Nov 2018 16:47:46 +0100
Subject: [PATCH 71/88] Fix traefik test

---
 metricbeat/module/traefik/test_traefik.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/metricbeat/module/traefik/test_traefik.py b/metricbeat/module/traefik/test_traefik.py
index be42e4d676f0..50a5df27670a 100644
--- a/metricbeat/module/traefik/test_traefik.py
+++ b/metricbeat/module/traefik/test_traefik.py
@@ -25,5 +25,4 @@ def test_health(self, metricset):
         self.check_metricset("traefik", metricset, self.get_hosts(), self.FIELDS + ["service.name"])
 
     def get_hosts(self):
-        return [os.getenv('TRAEFIK_HOST', self.compose_hosts()[0]) + ':' +
-                os.getenv('TRAEFIK_API_PORT', '8080')]
+        return [self.compose_host()]

From 58c4e28e6f716689a656d45d7979c8e14a32cb52 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 15 Nov 2018 20:11:20 +0100
Subject: [PATCH 72/88] Add also implementation for private port to python

---
 libbeat/tests/system/beat/compose.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index c322c4e34ce5..ad6b2e2f6f0d 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -95,6 +95,20 @@ def compose_down(cls):
         if INTEGRATION_TESTS and cls.COMPOSE_SERVICES:
             cls.compose_project().kill(service_names=cls.COMPOSE_SERVICES)
 
+    @classmethod
+    def _private_host(cls, info, port):
+        networks = info['NetworkSettings']['Networks'].values()
+        port = port.split("/")[0]
+        for network in networks:
+            ip = network['IPAddress']
+            if ip:
+                return "%s:%s" % (ip, port)
+
+    @classmethod
+    def _public_host(cls, info, port):
+        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
+        return "localhost:%s" % hostPort
+
     @classmethod
     def compose_host(cls, service=None, port=None):
         if not INTEGRATION_TESTS or not cls.COMPOSE_SERVICES:
@@ -109,13 +123,11 @@ def compose_host(cls, service=None, port=None):
 
         container = cls.compose_project().containers(service_names=[service])[0]
         info = container.inspect()
-
         if port is None:
             portsConfig = info['HostConfig']['PortBindings']
             port = portsConfig.keys()[0]
 
-        hostPort = info['NetworkSettings']['Ports'][port][0]['HostPort']
-        return "localhost:%s" % hostPort
+        return cls._public_host(info, port)
 
     @classmethod
     def compose_project(cls):

From 58981e54b610d4cde3e6197813a9e84b495618c1 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 20 Nov 2018 10:49:37 +0100
Subject: [PATCH 73/88] Better error message if no port is configured

---
 libbeat/tests/system/beat/compose.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/libbeat/tests/system/beat/compose.py b/libbeat/tests/system/beat/compose.py
index ad6b2e2f6f0d..cbbca55947c2 100644
--- a/libbeat/tests/system/beat/compose.py
+++ b/libbeat/tests/system/beat/compose.py
@@ -123,8 +123,10 @@ def compose_host(cls, service=None, port=None):
 
         container = cls.compose_project().containers(service_names=[service])[0]
         info = container.inspect()
+        portsConfig = info['HostConfig']['PortBindings']
+        if len(portsConfig) == 0:
+            raise Exception("No exposed ports for service %s" % service)
         if port is None:
-            portsConfig = info['HostConfig']['PortBindings']
             port = portsConfig.keys()[0]
 
         return cls._public_host(info, port)

From a0d8c3ecdd7bd686a483a2f31f91e21eed923261 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 20 Nov 2018 10:51:05 +0100
Subject: [PATCH 74/88] Fix etcd system tests

---
 metricbeat/docker-compose.yml       | 2 ++
 metricbeat/module/etcd/test_etcd.py | 3 +--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index 7e3f66d62fa0..8364e7eeec24 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -67,6 +67,8 @@ services:
       context: ./module/etcd/_meta
       args:
         ETCD_VERSION: v3.2.25
+    ports:
+      - 2379
 
   haproxy:
     build: ./module/haproxy/_meta
diff --git a/metricbeat/module/etcd/test_etcd.py b/metricbeat/module/etcd/test_etcd.py
index 9441c6e6d75b..981185947677 100644
--- a/metricbeat/module/etcd/test_etcd.py
+++ b/metricbeat/module/etcd/test_etcd.py
@@ -25,8 +25,7 @@ def test_metricset(self, metricset):
         self.check_metricset("etcd", metricset, self.get_hosts(), ['etcd.' + metricset])
 
     def get_hosts(self):
-        return [self.compose_hosts()[0] + ':' +
-                os.getenv('ETCD_PORT', '2379')]
+        return [self.compose_host()]
 
 
 class Test_3_2(Test):

From 95696e5f8662bbb3d8511db97c2f5387e326a11d Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Thu, 22 Nov 2018 16:22:48 +0100
Subject: [PATCH 75/88] Properly wrap testing.T into runner control

---
 libbeat/tests/compose/runner.go | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index d9746c62dc0e..233caf675cb1 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -79,9 +79,11 @@ func (r *TestRunner) runSuite(t *testing.T, tests Suite, ctl R) {
 		var testFunc func(t *testing.T)
 		switch f := test.(type) {
 		case func(R):
-			testFunc = func(t *testing.T) { f(ctl) }
+			testFunc = func(t *testing.T) { f(ctl.WithT(t)) }
 		case func(*testing.T, R):
-			testFunc = func(t *testing.T) { f(t, ctl) }
+			testFunc = func(t *testing.T) { f(t, ctl.WithT(t)) }
+		case func(*testing.T):
+			testFunc = func(t *testing.T) { f(t) }
 		default:
 			t.Fatalf("incorrect test suite function '%s'", name)
 		}
@@ -99,7 +101,6 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	t.Logf("Test host overriden by %s=%s", env, host)
 
 	ctl := &runnerControl{
-		T:    t,
 		host: host,
 	}
 	r.runSuite(t, tests, ctl)
@@ -168,7 +169,6 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 			}
 
 			ctl := &runnerControl{
-				T:        t,
 				host:     host,
 				scenario: s,
 			}
@@ -182,6 +182,8 @@ func (r *TestRunner) Run(t *testing.T, tests Suite) {
 type R interface {
 	testing.TB
 
+	WithT(t *testing.T) R
+
 	Host() string
 	Hostname() string
 	Port() string
@@ -196,6 +198,12 @@ type runnerControl struct {
 	scenario map[string]string
 }
 
+func (r *runnerControl) WithT(t *testing.T) R {
+	ctl := *r
+	ctl.T = t
+	return &ctl
+}
+
 func (r *runnerControl) Host() string {
 	return r.host
 }

From 6ba4e9d683eeb60e53dd5247d06d083c68967279 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 23 Nov 2018 12:56:05 +0100
Subject: [PATCH 76/88] Move compose test functions to top-level

---
 .../namespace/namespace_integration_test.go   |  18 ++-
 .../apache/status/status_integration_test.go  |  43 +++---
 .../collector/collector_integration_test.go   |  86 +++++------
 .../elasticsearch_integration_test.go         |  84 ++++++-----
 .../server/server_integration_test.go         |  38 ++---
 .../etcd/leader/leader_integration_test.go    |  38 ++---
 .../module/etcd/self/self_integration_test.go |  38 ++---
 .../etcd/store/store_integration_test.go      |  38 ++---
 .../module/http/json/json_integration_test.go |  54 ++++---
 .../jolokia/jmx/jmx_integration_test.go       |  48 +++---
 .../consumergroup_integration_test.go         |  40 ++---
 .../partition/partition_integration_test.go   | 139 +++++++++---------
 .../kibana/stats/stats_integration_test.go    | 102 +++++++------
 .../kibana/status/status_integration_test.go  |  44 +++---
 .../logstash/logstash_integration_test.go     |  50 ++++---
 .../memcached/stats/stats_integration_test.go |  16 +-
 .../collstats/collstats_integration_test.go   |  54 +++----
 .../dbstats/dbstats_integration_test.go       | 104 ++++++-------
 .../metrics/metrics_intergration_test.go      |  50 ++++---
 .../replstatus/replstatus_integration_test.go |  89 +++++------
 .../mongodb/status/status_integration_test.go |  56 +++----
 .../module/mysql/mysql_integration_test.go    |  16 +-
 .../mysql/status/status_integration_test.go   | 110 +++++++-------
 .../stubstatus/stubstatus_integration_test.go |  40 ++---
 .../php_fpm/pool/pool_integration_test.go     |  16 +-
 .../process/process_integration_test.go       |  16 +-
 .../activity/activity_integration_test.go     |  58 ++++----
 .../bgwriter/bgwriter_integration_test.go     |  64 ++++----
 .../database/database_integration_test.go     |  64 ++++----
 .../statement/statement_integration_test.go   | 134 +++++++++--------
 .../collector/collector_integration_test.go   |  38 ++---
 .../stats/stats_integration_test.go           |  46 +++---
 .../connection/connection_integration_test.go |  16 +-
 .../exchange/exchange_integration_test.go     |  24 +--
 .../rabbitmq/node/node_integration_test.go    |  16 +-
 .../rabbitmq/queue/queue_integration_test.go  |  22 +--
 .../redis/info/info_integration_test.go       | 118 ++++++++-------
 .../keyspace/keyspace_integration_test.go     |  70 ++++-----
 .../traefik/health/health_integration_test.go |  60 ++++----
 .../uwsgi/status/status_integration_test.go   |  44 +++---
 .../zookeeper/mntr/mntr_integration_test.go   |  57 +++----
 41 files changed, 1202 insertions(+), 1056 deletions(-)

diff --git a/metricbeat/module/aerospike/namespace/namespace_integration_test.go b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
index 9643548c699c..843af5450da9 100644
--- a/metricbeat/module/aerospike/namespace/namespace_integration_test.go
+++ b/metricbeat/module/aerospike/namespace/namespace_integration_test.go
@@ -28,15 +28,17 @@ import (
 )
 
 func TestNamespace(t *testing.T) {
-	t.Parallel()
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": testData,
+	})
+}
 
-	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-		err := mbtest.WriteEvents(f, t)
-		if err != nil {
-			t.Fatal("write", err)
-		}
-	}})
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/apache/status/status_integration_test.go b/metricbeat/module/apache/status/status_integration_test.go
index 631bd0cc0b15..0fbdd22a4101 100644
--- a/metricbeat/module/apache/status/status_integration_test.go
+++ b/metricbeat/module/apache/status/status_integration_test.go
@@ -31,30 +31,33 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+		"Data":  testData,
+		"Fetch": testFetch,
+	})
+}
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
 
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			// Check number of fields.
-			if len(event) < 11 {
-				t.Fatal("Too few top-level elements in the event")
-			}
-		},
-	})
+	// Check number of fields.
+	if len(event) < 11 {
+		t.Fatal("Too few top-level elements in the event")
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/dropwizard/collector/collector_integration_test.go b/metricbeat/module/dropwizard/collector/collector_integration_test.go
index 8f4af491b007..f864cabe2101 100644
--- a/metricbeat/module/dropwizard/collector/collector_integration_test.go
+++ b/metricbeat/module/dropwizard/collector/collector_integration_test.go
@@ -35,55 +35,59 @@ func TestCollector(t *testing.T) {
 	runner := compose.TestRunner{Service: "dropwizard"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			events, err := f.Fetch()
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			hasTag := false
-			doesntHaveTag := false
-			for _, event := range events {
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	events, err := f.Fetch()
 
-				ok, _ := event.HasKey("my_histogram")
-				if ok {
-					_, err := event.GetValue("tags")
-					if err == nil {
-						t.Fatal("write", "my_counter not supposed to have tags")
-					}
-					doesntHaveTag = true
-				}
+	hasTag := false
+	doesntHaveTag := false
+	for _, event := range events {
 
-				ok, _ = event.HasKey("my_counter")
-				if ok {
-					tagsRaw, err := event.GetValue("tags")
-					if err != nil {
-						t.Fatal("write", err)
-					} else {
-						tags, ok := tagsRaw.(common.MapStr)
-						if !ok {
-							t.Fatal("write", "unable to cast tags to common.MapStr")
-						} else {
-							assert.Equal(t, len(tags), 1)
-							hasTag = true
-						}
-					}
-				}
-			}
-			assert.Equal(t, hasTag, true)
-			assert.Equal(t, doesntHaveTag, true)
-			if !assert.NoError(t, err) {
-				t.FailNow()
+		ok, _ := event.HasKey("my_histogram")
+		if ok {
+			_, err := event.GetValue("tags")
+			if err == nil {
+				t.Fatal("write", "my_counter not supposed to have tags")
 			}
+			doesntHaveTag = true
+		}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvents(f, t)
+		ok, _ = event.HasKey("my_counter")
+		if ok {
+			tagsRaw, err := event.GetValue("tags")
 			if err != nil {
 				t.Fatal("write", err)
+			} else {
+				tags, ok := tagsRaw.(common.MapStr)
+				if !ok {
+					t.Fatal("write", "unable to cast tags to common.MapStr")
+				} else {
+					assert.Equal(t, len(tags), 1)
+					hasTag = true
+				}
 			}
-		},
-	})
+		}
+	}
+	assert.Equal(t, hasTag, true)
+	assert.Equal(t, doesntHaveTag, true)
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index ec4cd3a3a964..311d964c3209 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -72,49 +72,53 @@ func TestElasticsearch(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
-				t.Skip("This test fails on this version")
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
+
+func testFetch(t *testing.T, r compose.R) {
+	if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+		t.Skip("This test fails on this version")
+	}
+
+	host := r.Host()
+	err := createIndex(host)
+	assert.NoError(t, err)
+
+	err = enableTrialLicense(host)
+	assert.NoError(t, err)
+
+	err = createMLJob(host)
+	assert.NoError(t, err)
 
-			host := r.Host()
-			err := createIndex(host)
-			assert.NoError(t, err)
-
-			err = enableTrialLicense(host)
-			assert.NoError(t, err)
-
-			err = createMLJob(host)
-			assert.NoError(t, err)
-
-			for _, metricSet := range metricSets {
-				t.Run(metricSet, func(t *testing.T) {
-					checkSkip(t, metricSet, host)
-					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
-					events, errs := mbtest.ReportingFetchV2(f)
-
-					assert.Empty(t, errs)
-					if !assert.NotEmpty(t, events) {
-						t.FailNow()
-					}
-					t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-						events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
-				})
+	for _, metricSet := range metricSets {
+		t.Run(metricSet, func(t *testing.T) {
+			checkSkip(t, metricSet, host)
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, host))
+			events, errs := mbtest.ReportingFetchV2(f)
+
+			assert.Empty(t, errs)
+			if !assert.NotEmpty(t, events) {
+				t.FailNow()
 			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				t.Run(metricSet, func(t *testing.T) {
-					checkSkip(t, metricSet, r.Host())
-					f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
-					err := mbtest.WriteEventsReporterV2(f, t, metricSet)
-					if err != nil {
-						t.Fatal("write", err)
-					}
-				})
+			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+				events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint())
+		})
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		t.Run(metricSet, func(t *testing.T) {
+			checkSkip(t, metricSet, r.Host())
+			f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet, r.Host()))
+			err := mbtest.WriteEventsReporterV2(f, t, metricSet)
+			if err != nil {
+				t.Fatal("write", err)
 			}
-		},
-	})
+		})
+	}
 }
 
 // GetConfig returns config for elasticsearch module
diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go
index 385732d257ca..00e8b79a9c3c 100644
--- a/metricbeat/module/envoyproxy/server/server_integration_test.go
+++ b/metricbeat/module/envoyproxy/server/server_integration_test.go
@@ -32,26 +32,30 @@ func TestEnvoyproxy(t *testing.T) {
 	runner := compose.TestRunner{Service: "envoyproxy"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "envoyproxy",
diff --git a/metricbeat/module/etcd/leader/leader_integration_test.go b/metricbeat/module/etcd/leader/leader_integration_test.go
index bd2dd7bc519c..82583abc450b 100644
--- a/metricbeat/module/etcd/leader/leader_integration_test.go
+++ b/metricbeat/module/etcd/leader/leader_integration_test.go
@@ -32,26 +32,30 @@ func TestLeader(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/etcd/self/self_integration_test.go b/metricbeat/module/etcd/self/self_integration_test.go
index be8b979f7c13..c8ef9f1726b7 100644
--- a/metricbeat/module/etcd/self/self_integration_test.go
+++ b/metricbeat/module/etcd/self/self_integration_test.go
@@ -32,26 +32,30 @@ func TestLeader(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/etcd/store/store_integration_test.go b/metricbeat/module/etcd/store/store_integration_test.go
index 22be4797e8c1..53dc6a41afdb 100644
--- a/metricbeat/module/etcd/store/store_integration_test.go
+++ b/metricbeat/module/etcd/store/store_integration_test.go
@@ -32,26 +32,30 @@ func TestStore(t *testing.T) {
 	runner := compose.TestRunner{Service: "etcd"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "etcd",
diff --git a/metricbeat/module/http/json/json_integration_test.go b/metricbeat/module/http/json/json_integration_test.go
index 1e91c5ded020..85748e0a843c 100644
--- a/metricbeat/module/http/json/json_integration_test.go
+++ b/metricbeat/module/http/json/json_integration_test.go
@@ -32,32 +32,38 @@ func TestJSON(t *testing.T) {
 	runner := compose.TestRunner{Service: "http"}
 
 	runner.Run(t, compose.Suite{
-		"FetchObject": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"FetchObject": testFetchObject,
+		"FetchArray":  testFetchArray,
+		"Data":        testData,
+	})
+}
+
+func testFetchObject(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"FetchArray": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+func testFetchArray(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("array", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig("object", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(jsonType string, host string) map[string]interface{} {
diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
index d28146144e68..b7ba62f37428 100644
--- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go
+++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go
@@ -32,31 +32,35 @@ func TestJMX(t *testing.T) {
 	runner := compose.TestRunner{Service: "jolokia"}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			for _, config := range getConfigs(r.Host()) {
-				f := mbtest.NewEventsFetcher(t, config)
-				events, err := f.Fetch()
-				if !assert.NoError(t, err) {
-					t.FailNow()
-				}
-				t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
-				if len(events) == 0 || len(events[0]) <= 1 {
-					t.Fatal("Empty events")
-				}
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, config := range getConfigs(r.Host()) {
-				f := mbtest.NewEventsFetcher(t, config)
-				err := mbtest.WriteEvents(f, t)
-				if err != nil {
-					t.Fatal("write", err)
-				}
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	for _, config := range getConfigs(r.Host()) {
+		f := mbtest.NewEventsFetcher(t, config)
+		events, err := f.Fetch()
+		if !assert.NoError(t, err) {
+			t.FailNow()
+		}
+		t.Logf("%s/%s events: %+v", f.Module().Name(), f.Name(), events)
+		if len(events) == 0 || len(events[0]) <= 1 {
+			t.Fatal("Empty events")
+		}
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, config := range getConfigs(r.Host()) {
+		f := mbtest.NewEventsFetcher(t, config)
+		err := mbtest.WriteEvents(f, t)
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}
+}
+
 func getConfigs(host string) []map[string]interface{} {
 	return []map[string]interface{}{
 		{
diff --git a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
index e5a3e1f20b79..e2da5be0658c 100644
--- a/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
+++ b/metricbeat/module/kafka/consumergroup/consumergroup_integration_test.go
@@ -31,25 +31,29 @@ import (
 )
 
 func TestConsumerGroup(t *testing.T) {
-	mtest.Runner.Run(t, compose.Suite{"Data": func(t *testing.T, r compose.R) {
-		topic := "metricbeat-test"
-		mtest.GenerateKafkaData(t, topic, r.Host())
-		c, err := mtest.StartConsumer(t, topic, r.Host())
-		if err != nil {
-			t.Fatal(errors.Wrap(err, "starting kafka consumer"))
-		}
-		defer c.Close()
-
-		ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
-		for retries := 0; retries < 3; retries++ {
-			err = mbtest.WriteEventsReporterV2(ms, t, "")
-			if err == nil {
-				return
-			}
-			time.Sleep(500 * time.Millisecond)
+	mtest.Runner.Run(t, compose.Suite{
+		"Data": testData,
+	})
+}
+
+func testData(t *testing.T, r compose.R) {
+	topic := "metricbeat-test"
+	mtest.GenerateKafkaData(t, topic, r.Host())
+	c, err := mtest.StartConsumer(t, topic, r.Host())
+	if err != nil {
+		t.Fatal(errors.Wrap(err, "starting kafka consumer"))
+	}
+	defer c.Close()
+
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+	for retries := 0; retries < 3; retries++ {
+		err = mbtest.WriteEventsReporterV2(ms, t, "")
+		if err == nil {
+			return
 		}
-		t.Fatal("write", err)
-	}})
+		time.Sleep(500 * time.Millisecond)
+	}
+	t.Fatal("write", err)
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/kafka/partition/partition_integration_test.go b/metricbeat/module/kafka/partition/partition_integration_test.go
index 1ef103a208f7..ace9f55e7cb0 100644
--- a/metricbeat/module/kafka/partition/partition_integration_test.go
+++ b/metricbeat/module/kafka/partition/partition_integration_test.go
@@ -41,73 +41,78 @@ func TestPartition(t *testing.T) {
 	logp.TestingSetup(logp.WithSelectors("kafka"))
 
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
-
-			ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Topic": func(t *testing.T, r compose.R) {
-			id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
-			testTopic := fmt.Sprintf("test-metricbeat-%s", id)
-
-			// Create initial topic
-			mtest.GenerateKafkaData(t, testTopic, r.Host())
-
-			f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
-			dataBefore, err := mbtest.ReportingFetchV2(f)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-			if len(dataBefore) == 0 {
-				t.Errorf("No offsets fetched from topic (before): %v", testTopic)
-			}
-			t.Logf("before: %v", dataBefore)
-
-			var n int64 = 10
-			// Create n messages
-			for i := int64(0); i < n; i++ {
-				mtest.GenerateKafkaData(t, testTopic, r.Host())
-			}
-
-			dataAfter, err := mbtest.ReportingFetchV2(f)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-			if len(dataAfter) == 0 {
-				t.Errorf("No offsets fetched from topic (after): %v", testTopic)
-			}
-			t.Logf("after: %v", dataAfter)
-
-			// Checks that no new topics / partitions were added
-			assert.True(t, len(dataBefore) == len(dataAfter))
-
-			var offsetBefore int64 = 0
-			var offsetAfter int64 = 0
-
-			// Its possible that other topics exists -> select the right data
-			for _, data := range dataBefore {
-				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-					offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
-				}
-			}
-
-			for _, data := range dataAfter {
-				if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
-					offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
-				}
-			}
-
-			// Compares offset before and after
-			if offsetBefore+n != offsetAfter {
-				t.Errorf("Offset before: %v", offsetBefore)
-				t.Errorf("Offset after: %v", offsetAfter)
-			}
-			assert.True(t, offsetBefore+n == offsetAfter)
-		}})
+		"Data":  testData,
+		"Topic": testTopic,
+	})
+}
+
+func testData(t *testing.T, r compose.R) {
+	mtest.GenerateKafkaData(t, "metricbeat-generate-data", r.Host())
+
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig("", r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func testTopic(t *testing.T, r compose.R) {
+	id := strconv.Itoa(rand.New(rand.NewSource(int64(time.Now().Nanosecond()))).Int())
+	testTopic := fmt.Sprintf("test-metricbeat-%s", id)
+
+	// Create initial topic
+	mtest.GenerateKafkaData(t, testTopic, r.Host())
+
+	f := mbtest.NewReportingMetricSetV2(t, getConfig(testTopic, r.Host()))
+	dataBefore, err := mbtest.ReportingFetchV2(f)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+	if len(dataBefore) == 0 {
+		t.Errorf("No offsets fetched from topic (before): %v", testTopic)
+	}
+	t.Logf("before: %v", dataBefore)
+
+	var n int64 = 10
+	// Create n messages
+	for i := int64(0); i < n; i++ {
+		mtest.GenerateKafkaData(t, testTopic, r.Host())
+	}
+
+	dataAfter, err := mbtest.ReportingFetchV2(f)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+	if len(dataAfter) == 0 {
+		t.Errorf("No offsets fetched from topic (after): %v", testTopic)
+	}
+	t.Logf("after: %v", dataAfter)
+
+	// Checks that no new topics / partitions were added
+	assert.True(t, len(dataBefore) == len(dataAfter))
+
+	var offsetBefore int64 = 0
+	var offsetAfter int64 = 0
+
+	// Its possible that other topics exists -> select the right data
+	for _, data := range dataBefore {
+		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+			offsetBefore = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		}
+	}
+
+	for _, data := range dataAfter {
+		if data.ModuleFields["topic"].(common.MapStr)["name"] == testTopic {
+			offsetAfter = data.MetricSetFields["offset"].(common.MapStr)["newest"].(int64)
+		}
+	}
+
+	// Compares offset before and after
+	if offsetBefore+n != offsetAfter {
+		t.Errorf("Offset before: %v", offsetBefore)
+		t.Errorf("Offset after: %v", offsetAfter)
+	}
+	assert.True(t, offsetBefore+n == offsetAfter)
 }
 
 func getConfig(topic string, host string) map[string]interface{} {
diff --git a/metricbeat/module/kibana/stats/stats_integration_test.go b/metricbeat/module/kibana/stats/stats_integration_test.go
index 0dd68e443692..8872e975c8b1 100644
--- a/metricbeat/module/kibana/stats/stats_integration_test.go
+++ b/metricbeat/module/kibana/stats/stats_integration_test.go
@@ -37,58 +37,62 @@ import (
 
 func TestStat(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("stats", r.Host())
-			version, err := getKibanaVersion(r.Host())
-			if err != nil {
-				t.Fatal("getting kibana version", err)
-			}
-
-			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-			if err != nil {
-				t.Fatal("checking if kibana stats API is available", err)
-			}
-
-			if !isStatsAPIAvailable {
-				t.Skip("Kibana stats API is not available until 6.4.0")
-			}
-
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("kibana", "stats").Fields.StringToPrint())
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("stats", r.Host())
-			version, err := getKibanaVersion(r.Host())
-			if err != nil {
-				t.Fatal("getting kibana version", err)
-			}
-
-			isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
-			if err != nil {
-				t.Fatal("checking if kibana stats API is available", err)
-			}
-
-			if !isStatsAPIAvailable {
-				t.Skip("Kibana stats API is not available until 6.4.0")
-			}
-
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			err = mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("stats", r.Host())
+	version, err := getKibanaVersion(r.Host())
+	if err != nil {
+		t.Fatal("getting kibana version", err)
+	}
+
+	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+	if err != nil {
+		t.Fatal("checking if kibana stats API is available", err)
+	}
+
+	if !isStatsAPIAvailable {
+		t.Skip("Kibana stats API is not available until 6.4.0")
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("kibana", "stats").Fields.StringToPrint())
+}
+
+func testData(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("stats", r.Host())
+	version, err := getKibanaVersion(r.Host())
+	if err != nil {
+		t.Fatal("getting kibana version", err)
+	}
+
+	isStatsAPIAvailable, err := kibana.IsStatsAPIAvailable(version)
+	if err != nil {
+		t.Fatal("checking if kibana stats API is available", err)
+	}
+
+	if !isStatsAPIAvailable {
+		t.Skip("Kibana stats API is not available until 6.4.0")
+	}
+
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	err = mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getKibanaVersion(kibanaHostPort string) (string, error) {
 	resp, err := http.Get("http://" + kibanaHostPort + "/api/status")
 	if err != nil {
diff --git a/metricbeat/module/kibana/status/status_integration_test.go b/metricbeat/module/kibana/status/status_integration_test.go
index d86857b91e9b..f8dcb36fef8c 100644
--- a/metricbeat/module/kibana/status/status_integration_test.go
+++ b/metricbeat/module/kibana/status/status_integration_test.go
@@ -31,25 +31,29 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host()))
-			events, errs := mbtest.ReportingFetchV2(f)
-
-			assert.Empty(t, errs)
-			if !assert.NotEmpty(t, events) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-				events[0].BeatEvent("kibana", "status").Fields.StringToPrint())
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("status", r.Host())
-			f := mbtest.NewReportingMetricSetV2(t, config)
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("status", r.Host()))
+	events, errs := mbtest.ReportingFetchV2(f)
+
+	assert.Empty(t, errs)
+	if !assert.NotEmpty(t, events) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+		events[0].BeatEvent("kibana", "status").Fields.StringToPrint())
+}
+
+func testData(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("status", r.Host())
+	f := mbtest.NewReportingMetricSetV2(t, config)
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/logstash/logstash_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go
index 8997f86709bd..1e0e8a9fe8a5 100644
--- a/metricbeat/module/logstash/logstash_integration_test.go
+++ b/metricbeat/module/logstash/logstash_integration_test.go
@@ -38,29 +38,33 @@ var metricSets = []string{
 
 func TestLogstash(t *testing.T) {
 	logstash.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet, r.Host()))
-				events, errs := mbtest.ReportingFetchV2(f)
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-				assert.Empty(t, errs)
-				if !assert.NotEmpty(t, events) {
-					t.FailNow()
-				}
+func testFetch(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet, r.Host()))
+		events, errs := mbtest.ReportingFetchV2(f)
 
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
-					events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint())
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			for _, metricSet := range metricSets {
-				config := logstash.GetConfig(metricSet, r.Host())
-				f := mbtest.NewReportingMetricSetV2(t, config)
-				err := mbtest.WriteEventsReporterV2(f, t, "")
-				if err != nil {
-					t.Fatal("write", err)
-				}
-			}
-		},
-	})
+		assert.Empty(t, errs)
+		if !assert.NotEmpty(t, events) {
+			t.FailNow()
+		}
+
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(),
+			events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint())
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	for _, metricSet := range metricSets {
+		config := logstash.GetConfig(metricSet, r.Host())
+		f := mbtest.NewReportingMetricSetV2(t, config)
+		err := mbtest.WriteEventsReporterV2(f, t, "")
+		if err != nil {
+			t.Fatal("write", err)
+		}
+	}
 }
diff --git a/metricbeat/module/memcached/stats/stats_integration_test.go b/metricbeat/module/memcached/stats/stats_integration_test.go
index 1672d600cb0e..29a513189bfa 100644
--- a/metricbeat/module/memcached/stats/stats_integration_test.go
+++ b/metricbeat/module/memcached/stats/stats_integration_test.go
@@ -30,16 +30,18 @@ func TestStats(t *testing.T) {
 	runner := compose.TestRunner{Service: "memcached"}
 
 	runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "memcached",
diff --git a/metricbeat/module/mongodb/collstats/collstats_integration_test.go b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
index 98b5b0b5028f..53aca599748d 100644
--- a/metricbeat/module/mongodb/collstats/collstats_integration_test.go
+++ b/metricbeat/module/mongodb/collstats/collstats_integration_test.go
@@ -31,30 +31,34 @@ import (
 
 func TestCollstats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			for _, event := range events {
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-				// Check a few event Fields
-				db := event["db"].(string)
-				assert.NotEqual(t, db, "")
-
-				collection := event["collection"].(string)
-				assert.NotEqual(t, collection, "")
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+		// Check a few event Fields
+		db := event["db"].(string)
+		assert.NotEqual(t, db, "")
+
+		collection := event["collection"].(string)
+		assert.NotEqual(t, collection, "")
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("collstats", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
index 8cea2158fe1e..e794495c0579 100644
--- a/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
+++ b/metricbeat/module/mongodb/dbstats/dbstats_integration_test.go
@@ -31,55 +31,59 @@ import (
 
 func TestDBStats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			for _, event := range events {
-				t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-				// Check a few event Fields
-				db := event["db"].(string)
-				assert.NotEqual(t, db, "")
-
-				collections := event["collections"].(int64)
-				assert.True(t, collections > 0)
-
-				objects := event["objects"].(int64)
-				assert.True(t, objects > 0)
-
-				avgObjSize, err := event.GetValue("avg_obj_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, avgObjSize.(int64) > 0)
-
-				dataSize, err := event.GetValue("data_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, dataSize.(int64) > 0)
-
-				storageSize, err := event.GetValue("storage_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, storageSize.(int64) > 0)
-
-				numExtents := event["num_extents"].(int64)
-				assert.True(t, numExtents >= 0)
-
-				indexes := event["indexes"].(int64)
-				assert.True(t, indexes >= 0)
-
-				indexSize, err := event.GetValue("index_size.bytes")
-				assert.NoError(t, err)
-				assert.True(t, indexSize.(int64) > 0)
-			}
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	for _, event := range events {
+		t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+		// Check a few event Fields
+		db := event["db"].(string)
+		assert.NotEqual(t, db, "")
+
+		collections := event["collections"].(int64)
+		assert.True(t, collections > 0)
+
+		objects := event["objects"].(int64)
+		assert.True(t, objects > 0)
+
+		avgObjSize, err := event.GetValue("avg_obj_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, avgObjSize.(int64) > 0)
+
+		dataSize, err := event.GetValue("data_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, dataSize.(int64) > 0)
+
+		storageSize, err := event.GetValue("storage_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, storageSize.(int64) > 0)
+
+		numExtents := event["num_extents"].(int64)
+		assert.True(t, numExtents >= 0)
+
+		indexes := event["indexes"].(int64)
+		assert.True(t, indexes >= 0)
+
+		indexSize, err := event.GetValue("index_size.bytes")
+		assert.NoError(t, err)
+		assert.True(t, indexSize.(int64) > 0)
+	}
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("dbstats", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
index 0d706716a403..5126810e1abf 100644
--- a/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
+++ b/metricbeat/module/mongodb/metrics/metrics_intergration_test.go
@@ -31,28 +31,32 @@ import (
 
 func TestMetrics(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			// Check a few event Fields
-			findCount, err := event.GetValue("commands.find.total")
-			assert.NoError(t, err)
-			assert.True(t, findCount.(int64) >= 0)
-
-			deletedDocuments, err := event.GetValue("document.deleted")
-			assert.NoError(t, err)
-			assert.True(t, deletedDocuments.(int64) >= 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	// Check a few event Fields
+	findCount, err := event.GetValue("commands.find.total")
+	assert.NoError(t, err)
+	assert.True(t, findCount.(int64) >= 0)
+
+	deletedDocuments, err := event.GetValue("document.deleted")
+	assert.NoError(t, err)
+	assert.True(t, deletedDocuments.(int64) >= 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("metrics", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
index c1a9b721b4d5..6db3179fc80d 100644
--- a/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
+++ b/metricbeat/module/mongodb/replstatus/replstatus_integration_test.go
@@ -36,52 +36,55 @@ import (
 
 func TestReplStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			err := initiateReplicaSet(t, r.Host())
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			oplog := event["oplog"].(common.MapStr)
-			allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
-			assert.True(t, allocated >= 0)
-
-			used := oplog["size"].(common.MapStr)["used"].(float64)
-			assert.True(t, used > 0)
-
-			firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
-			assert.True(t, firstTs >= 0)
-
-			window := oplog["window"].(int64)
-			assert.True(t, window >= 0)
-
-			members := event["members"].(common.MapStr)
-			primary := members["primary"].(common.MapStr)
-			assert.NotEmpty(t, primary["host"].(string))
-			assert.True(t, primary["optime"].(int64) > 0)
-
-			set := event["set_name"].(string)
-			assert.Equal(t, set, "beats")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	err := initiateReplicaSet(t, r.Host())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	oplog := event["oplog"].(common.MapStr)
+	allocated := oplog["size"].(common.MapStr)["allocated"].(int64)
+	assert.True(t, allocated >= 0)
+
+	used := oplog["size"].(common.MapStr)["used"].(float64)
+	assert.True(t, used > 0)
+
+	firstTs := oplog["first"].(common.MapStr)["timestamp"].(int64)
+	assert.True(t, firstTs >= 0)
+
+	window := oplog["window"].(int64)
+	assert.True(t, window >= 0)
+
+	members := event["members"].(common.MapStr)
+	primary := members["primary"].(common.MapStr)
+	assert.NotEmpty(t, primary["host"].(string))
+	assert.True(t, primary["optime"].(int64) > 0)
+
+	set := event["set_name"].(string)
+	assert.Equal(t, set, "beats")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("replstatus", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
 func initiateReplicaSet(t *testing.T, url string) error {
 	dialInfo, err := mgo.ParseURL(url)
 	if err != nil {
diff --git a/metricbeat/module/mongodb/status/status_integration_test.go b/metricbeat/module/mongodb/status/status_integration_test.go
index 91eab6daf6ce..cb19832f4893 100644
--- a/metricbeat/module/mongodb/status/status_integration_test.go
+++ b/metricbeat/module/mongodb/status/status_integration_test.go
@@ -32,31 +32,35 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			current := event["connections"].(common.MapStr)["current"].(int64)
-			assert.True(t, current >= 0)
-
-			available := event["connections"].(common.MapStr)["available"].(int64)
-			assert.True(t, available > 0)
-
-			pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
-			assert.True(t, pageFaults >= 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	current := event["connections"].(common.MapStr)["current"].(int64)
+	assert.True(t, current >= 0)
+
+	available := event["connections"].(common.MapStr)["available"].(int64)
+	assert.True(t, available > 0)
+
+	pageFaults := event["extra_info"].(common.MapStr)["page_faults"].(int64)
+	assert.True(t, pageFaults >= 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host()))
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/mysql/mysql_integration_test.go b/metricbeat/module/mysql/mysql_integration_test.go
index 602b617d215a..47217fae02e2 100644
--- a/metricbeat/module/mysql/mysql_integration_test.go
+++ b/metricbeat/module/mysql/mysql_integration_test.go
@@ -30,12 +30,14 @@ import (
 
 func TestNewDB(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"NewDB": func(t *testing.T, r compose.R) {
-			db, err := NewDB(mtest.GetDSN(r.Host()))
-			assert.NoError(t, err)
-
-			err = db.Ping()
-			assert.NoError(t, err)
-		},
+		"NewDB": testNewDB,
 	})
 }
+
+func testNewDB(t *testing.T, r compose.R) {
+	db, err := NewDB(mtest.GetDSN(r.Host()))
+	assert.NoError(t, err)
+
+	err = db.Ping()
+	assert.NoError(t, err)
+}
diff --git a/metricbeat/module/mysql/status/status_integration_test.go b/metricbeat/module/mysql/status/status_integration_test.go
index 67f1c8742359..a6b72b3488df 100644
--- a/metricbeat/module/mysql/status/status_integration_test.go
+++ b/metricbeat/module/mysql/status/status_integration_test.go
@@ -32,57 +32,63 @@ import (
 
 func TestStatus(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			connections := event["connections"].(int64)
-			open := event["open"].(common.MapStr)
-			openTables := open["tables"].(int64)
-			openFiles := open["files"].(int64)
-			openStreams := open["streams"].(int64)
-
-			assert.True(t, connections > 0)
-			assert.True(t, openTables > 0)
-			assert.True(t, openFiles >= 0)
-			assert.True(t, openStreams == 0)
-		},
-		"FetchRaw": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
-			assert.True(t, cachedThreads >= 0)
-
-			rawData := event["raw"].(common.MapStr)
-
-			// Make sure field was removed from raw fields as in schema
-			_, exists := rawData["Threads_cached"]
-			assert.False(t, exists)
-
-			// Check a raw field if it is available
-			_, exists = rawData["Slow_launch_threads"]
-			assert.True(t, exists)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch":    testFetch,
+		"FetchRaw": testFetchRaw,
+		"Data":     testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	connections := event["connections"].(int64)
+	open := event["open"].(common.MapStr)
+	openTables := open["tables"].(int64)
+	openFiles := open["files"].(int64)
+	openStreams := open["streams"].(int64)
+
+	assert.True(t, connections > 0)
+	assert.True(t, openTables > 0)
+	assert.True(t, openFiles >= 0)
+	assert.True(t, openStreams == 0)
+}
+
+func testFetchRaw(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), true))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	cachedThreads := event["threads"].(common.MapStr)["cached"].(int64)
+	assert.True(t, cachedThreads >= 0)
+
+	rawData := event["raw"].(common.MapStr)
+
+	// Make sure field was removed from raw fields as in schema
+	_, exists := rawData["Threads_cached"]
+	assert.False(t, exists)
+
+	// Check a raw field if it is available
+	_, exists = rawData["Slow_launch_threads"]
+	assert.True(t, exists)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("status", r.Host(), false))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 7c2a441b97a8..68d998bcde67 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -35,27 +35,31 @@ func TestStubstatus(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": tesFetch,
+		"Data":  testData,
+	})
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			// Check number of fields.
-			assert.Equal(t, 10, len(event))
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check number of fields.
+	assert.Equal(t, 10, len(event))
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {
diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go
index 332bb889a102..5c8489f4de59 100644
--- a/metricbeat/module/php_fpm/pool/pool_integration_test.go
+++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go
@@ -29,12 +29,14 @@ import (
 
 func TestPool(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("pool", r.Host()))
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/php_fpm/process/process_integration_test.go b/metricbeat/module/php_fpm/process/process_integration_test.go
index 66aa94e3acdd..55dcd22c877c 100644
--- a/metricbeat/module/php_fpm/process/process_integration_test.go
+++ b/metricbeat/module/php_fpm/process/process_integration_test.go
@@ -29,12 +29,14 @@ import (
 
 func TestProcess(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
-			err := mbtest.WriteEventsReporterV2(f, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("process", r.Host()))
+	err := mbtest.WriteEventsReporterV2(f, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/postgresql/activity/activity_integration_test.go b/metricbeat/module/postgresql/activity/activity_integration_test.go
index c48b399a8187..7533986cd226 100644
--- a/metricbeat/module/postgresql/activity/activity_integration_test.go
+++ b/metricbeat/module/postgresql/activity/activity_integration_test.go
@@ -32,37 +32,41 @@ import (
 
 func TestActivity(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			assert.True(t, len(events) > 0)
-			event := events[0]
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	assert.True(t, len(events) > 0)
+	event := events[0]
 
-			// Check event fields
-			assert.Contains(t, event, "database")
-			db_oid := event["database"].(common.MapStr)["oid"].(int64)
-			assert.True(t, db_oid > 0)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			assert.Contains(t, event, "pid")
-			assert.True(t, event["pid"].(int64) > 0)
+	// Check event fields
+	assert.Contains(t, event, "database")
+	db_oid := event["database"].(common.MapStr)["oid"].(int64)
+	assert.True(t, db_oid > 0)
 
-			assert.Contains(t, event, "user")
-			assert.Contains(t, event["user"].(common.MapStr), "name")
-			assert.Contains(t, event["user"].(common.MapStr), "id")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+	assert.Contains(t, event, "pid")
+	assert.True(t, event["pid"].(int64) > 0)
 
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	assert.Contains(t, event, "user")
+	assert.Contains(t, event["user"].(common.MapStr), "name")
+	assert.Contains(t, event["user"].(common.MapStr), "id")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("activity", r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
index 3562d21d3f0a..0a307d165f88 100644
--- a/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
+++ b/metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go
@@ -32,39 +32,43 @@ import (
 
 func TestBgwriter(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			assert.Contains(t, event, "checkpoints")
-			assert.Contains(t, event, "buffers")
-			assert.Contains(t, event, "stats_reset")
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			checkpoints := event["checkpoints"].(common.MapStr)
-			assert.Contains(t, checkpoints, "scheduled")
-			assert.Contains(t, checkpoints, "requested")
-			assert.Contains(t, checkpoints, "times")
+	assert.Contains(t, event, "checkpoints")
+	assert.Contains(t, event, "buffers")
+	assert.Contains(t, event, "stats_reset")
 
-			buffers := event["buffers"].(common.MapStr)
-			assert.Contains(t, buffers, "checkpoints")
-			assert.Contains(t, buffers, "clean")
-			assert.Contains(t, buffers, "clean_full")
-			assert.Contains(t, buffers, "backend")
-			assert.Contains(t, buffers, "backend_fsync")
-			assert.Contains(t, buffers, "allocated")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+	checkpoints := event["checkpoints"].(common.MapStr)
+	assert.Contains(t, checkpoints, "scheduled")
+	assert.Contains(t, checkpoints, "requested")
+	assert.Contains(t, checkpoints, "times")
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	buffers := event["buffers"].(common.MapStr)
+	assert.Contains(t, buffers, "checkpoints")
+	assert.Contains(t, buffers, "clean")
+	assert.Contains(t, buffers, "clean_full")
+	assert.Contains(t, buffers, "backend")
+	assert.Contains(t, buffers, "backend_fsync")
+	assert.Contains(t, buffers, "allocated")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("bgwriter", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/database/database_integration_test.go b/metricbeat/module/postgresql/database/database_integration_test.go
index 84b2ecfb3fbe..f7685b65c8ad 100644
--- a/metricbeat/module/postgresql/database/database_integration_test.go
+++ b/metricbeat/module/postgresql/database/database_integration_test.go
@@ -32,38 +32,42 @@ import (
 
 func TestDatabase(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-			assert.True(t, len(events) > 0)
-			event := events[0]
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+	assert.True(t, len(events) > 0)
+	event := events[0]
 
-			// Check event fields
-			db_oid := event["oid"].(int64)
-			assert.True(t, db_oid > 0)
-			assert.Contains(t, event, "name")
-			_, ok := event["name"].(string)
-			assert.True(t, ok)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			rows := event["rows"].(common.MapStr)
-			assert.Contains(t, rows, "returned")
-			assert.Contains(t, rows, "fetched")
-			assert.Contains(t, rows, "inserted")
-			assert.Contains(t, rows, "updated")
-			assert.Contains(t, rows, "deleted")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check event fields
+	db_oid := event["oid"].(int64)
+	assert.True(t, db_oid > 0)
+	assert.Contains(t, event, "name")
+	_, ok := event["name"].(string)
+	assert.True(t, ok)
+
+	rows := event["rows"].(common.MapStr)
+	assert.Contains(t, rows, "returned")
+	assert.Contains(t, rows, "fetched")
+	assert.Contains(t, rows, "inserted")
+	assert.Contains(t, rows, "updated")
+	assert.Contains(t, rows, "deleted")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("database", r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
diff --git a/metricbeat/module/postgresql/statement/statement_integration_test.go b/metricbeat/module/postgresql/statement/statement_integration_test.go
index 5055161c3173..6a1acb967d17 100644
--- a/metricbeat/module/postgresql/statement/statement_integration_test.go
+++ b/metricbeat/module/postgresql/statement/statement_integration_test.go
@@ -32,70 +32,74 @@ import (
 
 func TestStatement(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
-			events, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			assert.True(t, len(events) > 0)
-			event := events[0]
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check event fields
-			assert.Contains(t, event, "user")
-			assert.Contains(t, event["user"].(common.MapStr), "id")
-
-			assert.Contains(t, event, "database")
-			db_oid := event["database"].(common.MapStr)["oid"].(int64)
-			assert.True(t, db_oid > 0)
-
-			assert.Contains(t, event, "query")
-			query := event["query"].(common.MapStr)
-			assert.Contains(t, query, "id")
-			assert.Contains(t, query, "text")
-			assert.Contains(t, query, "calls")
-			assert.Contains(t, query, "rows")
-
-			assert.Contains(t, query, "time")
-			time := query["time"].(common.MapStr)
-			assert.Contains(t, time, "total")
-			assert.Contains(t, time, "min")
-			assert.Contains(t, time, "max")
-			assert.Contains(t, time, "mean")
-			assert.Contains(t, time, "stddev")
-
-			assert.Contains(t, query["memory"], "shared")
-			memory := query["memory"].(common.MapStr)
-
-			assert.Contains(t, memory, "shared")
-			shared := memory["shared"].(common.MapStr)
-			assert.Contains(t, shared, "hit")
-			assert.Contains(t, shared, "read")
-			assert.Contains(t, shared, "dirtied")
-			assert.Contains(t, shared, "written")
-
-			assert.Contains(t, memory, "local")
-			local := memory["local"].(common.MapStr)
-			assert.Contains(t, local, "hit")
-			assert.Contains(t, local, "read")
-			assert.Contains(t, local, "dirtied")
-			assert.Contains(t, local, "written")
-
-			assert.Contains(t, memory, "temp")
-			temp := memory["temp"].(common.MapStr)
-			assert.Contains(t, temp, "read")
-			assert.Contains(t, temp, "written")
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+	events, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	assert.True(t, len(events) > 0)
+	event := events[0]
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check event fields
+	assert.Contains(t, event, "user")
+	assert.Contains(t, event["user"].(common.MapStr), "id")
+
+	assert.Contains(t, event, "database")
+	db_oid := event["database"].(common.MapStr)["oid"].(int64)
+	assert.True(t, db_oid > 0)
+
+	assert.Contains(t, event, "query")
+	query := event["query"].(common.MapStr)
+	assert.Contains(t, query, "id")
+	assert.Contains(t, query, "text")
+	assert.Contains(t, query, "calls")
+	assert.Contains(t, query, "rows")
+
+	assert.Contains(t, query, "time")
+	time := query["time"].(common.MapStr)
+	assert.Contains(t, time, "total")
+	assert.Contains(t, time, "min")
+	assert.Contains(t, time, "max")
+	assert.Contains(t, time, "mean")
+	assert.Contains(t, time, "stddev")
+
+	assert.Contains(t, query["memory"], "shared")
+	memory := query["memory"].(common.MapStr)
+
+	assert.Contains(t, memory, "shared")
+	shared := memory["shared"].(common.MapStr)
+	assert.Contains(t, shared, "hit")
+	assert.Contains(t, shared, "read")
+	assert.Contains(t, shared, "dirtied")
+	assert.Contains(t, shared, "written")
+
+	assert.Contains(t, memory, "local")
+	local := memory["local"].(common.MapStr)
+	assert.Contains(t, local, "hit")
+	assert.Contains(t, local, "read")
+	assert.Contains(t, local, "dirtied")
+	assert.Contains(t, local, "written")
+
+	assert.Contains(t, memory, "temp")
+	temp := memory["temp"].(common.MapStr)
+	assert.Contains(t, temp, "read")
+	assert.Contains(t, temp, "written")
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, mtest.GetConfig("statement", r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/prometheus/collector/collector_integration_test.go b/metricbeat/module/prometheus/collector/collector_integration_test.go
index 1669078e6437..cfa610cb8987 100644
--- a/metricbeat/module/prometheus/collector/collector_integration_test.go
+++ b/metricbeat/module/prometheus/collector/collector_integration_test.go
@@ -34,26 +34,30 @@ import (
 
 func TestCollector(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetConfig("collector", host)
 	config["namespace"] = "collector"
diff --git a/metricbeat/module/prometheus/stats/stats_integration_test.go b/metricbeat/module/prometheus/stats/stats_integration_test.go
index 0f18a5997530..6cc37b4d6c81 100644
--- a/metricbeat/module/prometheus/stats/stats_integration_test.go
+++ b/metricbeat/module/prometheus/stats/stats_integration_test.go
@@ -32,27 +32,31 @@ import (
 
 func TestStats(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check number of fields.
-			assert.Equal(t, 3, len(event))
-			assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check number of fields.
+	assert.Equal(t, 3, len(event))
+	assert.True(t, event["processes"].(common.MapStr)["open_fds"].(int64) > 0)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, mtest.GetConfig("stats", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/rabbitmq/connection/connection_integration_test.go b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
index 420e65d5ccee..e669f99dc5f5 100644
--- a/metricbeat/module/rabbitmq/connection/connection_integration_test.go
+++ b/metricbeat/module/rabbitmq/connection/connection_integration_test.go
@@ -29,16 +29,18 @@ import (
 
 func TestConnection(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"connection"}
diff --git a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
index da20d0e7c478..c28c510d0200 100644
--- a/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
+++ b/metricbeat/module/rabbitmq/exchange/exchange_integration_test.go
@@ -30,20 +30,22 @@ import (
 
 func TestExchange(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-				hasIn, _ := e.HasKey("messages.publish_in")
-				hasOut, _ := e.HasKey("messages.publish_out")
-				return hasIn && hasOut
-			})
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+		hasIn, _ := e.HasKey("messages.publish_in")
+		hasOut, _ := e.HasKey("messages.publish_out")
+		return hasIn && hasOut
+	})
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"exchange"}
diff --git a/metricbeat/module/rabbitmq/node/node_integration_test.go b/metricbeat/module/rabbitmq/node/node_integration_test.go
index ca0ae772b53a..26b97ea245ec 100644
--- a/metricbeat/module/rabbitmq/node/node_integration_test.go
+++ b/metricbeat/module/rabbitmq/node/node_integration_test.go
@@ -29,16 +29,18 @@ import (
 
 func TestNode(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	ms := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"node"}
diff --git a/metricbeat/module/rabbitmq/queue/queue_integration_test.go b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
index 7f84b903fe49..026fc2c9163c 100644
--- a/metricbeat/module/rabbitmq/queue/queue_integration_test.go
+++ b/metricbeat/module/rabbitmq/queue/queue_integration_test.go
@@ -30,19 +30,21 @@ import (
 
 func TestQueue(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
-				hasTotal, _ := e.HasKey("messages.total")
-				return hasTotal
-			})
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Data": testData,
 	})
 }
 
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	err := mbtest.WriteEventsCond(f, t, func(e common.MapStr) bool {
+		hasTotal, _ := e.HasKey("messages.total")
+		return hasTotal
+	})
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 func getConfig(host string) map[string]interface{} {
 	config := mtest.GetIntegrationConfig(host)
 	config["metricsets"] = []string{"queue"}
diff --git a/metricbeat/module/redis/info/info_integration_test.go b/metricbeat/module/redis/info/info_integration_test.go
index 7d17df5b380a..c3151929085d 100644
--- a/metricbeat/module/redis/info/info_integration_test.go
+++ b/metricbeat/module/redis/info/info_integration_test.go
@@ -36,67 +36,71 @@ const (
 )
 
 func TestInfo(t *testing.T) {
-	t.Parallel()
-
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-			event, err := f.Fetch()
-			if err != nil {
-				t.Fatal("fetch", err)
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
-
-			// Check fields
-			assert.Equal(t, 9, len(event))
-			server := event["server"].(common.MapStr)
-			assert.Equal(t, "standalone", server["mode"])
-		},
-		"Passwords": func(t *testing.T, r compose.R) {
-			// Add password and ensure it gets reset
-			defer func() {
-				err := resetPassword(r.Host(), password)
-				if err != nil {
-					t.Fatal("resetting password", err)
-				}
-			}()
-
-			err := addPassword(r.Host(), password)
-			if err != nil {
-				t.Fatal("adding password", err)
-			}
-
-			// Test Fetch metrics with missing password
-			f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
-			_, err = f.Fetch()
-			if assert.Error(t, err, "missing password") {
-				assert.Contains(t, err, "NOAUTH Authentication required.")
-			}
-
-			// Config redis and metricset with an invalid password
-			f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
-			_, err = f.Fetch()
-			if assert.Error(t, err, "invalid password") {
-				assert.Contains(t, err, "ERR invalid password")
-			}
-
-			// Config redis and metricset with a valid password
-			f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
-			_, err = f.Fetch()
-			assert.NoError(t, err, "valid password")
-		},
+		"Fetch":     testFetch,
+		"Data":      testData,
+		"Passwords": testPasswords,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+	event, err := f.Fetch()
+	if err != nil {
+		t.Fatal("fetch", err)
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+
+	// Check fields
+	assert.Equal(t, 9, len(event))
+	server := event["server"].(common.MapStr)
+	assert.Equal(t, "standalone", server["mode"])
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
+func testPasswords(t *testing.T, r compose.R) {
+	// Add password and ensure it gets reset
+	defer func() {
+		err := resetPassword(r.Host(), password)
+		if err != nil {
+			t.Fatal("resetting password", err)
+		}
+	}()
+
+	err := addPassword(r.Host(), password)
+	if err != nil {
+		t.Fatal("adding password", err)
+	}
+
+	// Test Fetch metrics with missing password
+	f := mbtest.NewEventFetcher(t, getConfig("", r.Host()))
+	_, err = f.Fetch()
+	if assert.Error(t, err, "missing password") {
+		assert.Contains(t, err, "NOAUTH Authentication required.")
+	}
+
+	// Config redis and metricset with an invalid password
+	f = mbtest.NewEventFetcher(t, getConfig("blah", r.Host()))
+	_, err = f.Fetch()
+	if assert.Error(t, err, "invalid password") {
+		assert.Contains(t, err, "ERR invalid password")
+	}
+
+	// Config redis and metricset with a valid password
+	f = mbtest.NewEventFetcher(t, getConfig(password, r.Host()))
+	_, err = f.Fetch()
+	assert.NoError(t, err, "valid password")
+}
+
 // addPassword will add a password to redis.
 func addPassword(host, pass string) error {
 	c, err := rd.Dial("tcp", host)
diff --git a/metricbeat/module/redis/keyspace/keyspace_integration_test.go b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
index 4109d3845684..08925d011f72 100644
--- a/metricbeat/module/redis/keyspace/keyspace_integration_test.go
+++ b/metricbeat/module/redis/keyspace/keyspace_integration_test.go
@@ -32,44 +32,46 @@ import (
 )
 
 func TestKeyspace(t *testing.T) {
-	t.Parallel()
-
 	mtest.Runner.Run(t, compose.Suite{
-		"Data": func(t *testing.T, r compose.R) {
-			addEntry(t, r.Host())
-
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-		"Fetch": func(t *testing.T, r compose.R) {
-			addEntry(t, r.Host())
-
-			// Fetch data
-			f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
-			events, err := f.Fetch()
-			if err != nil {
-				t.Fatal("fetch", err)
-			}
-
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
-
-			// Make sure at least 1 db keyspace exists
-			assert.True(t, len(events) > 0)
-
-			keyspace := events[0]
-
-			assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
-			assert.True(t, keyspace["expires"].(int64) >= 0)
-			assert.True(t, keyspace["keys"].(int64) >= 0)
-			assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
+func testFetch(t *testing.T, r compose.R) {
+	addEntry(t, r.Host())
+
+	// Fetch data
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+	events, err := f.Fetch()
+	if err != nil {
+		t.Fatal("fetch", err)
+	}
+
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events)
+
+	// Make sure at least 1 db keyspace exists
+	assert.True(t, len(events) > 0)
+
+	keyspace := events[0]
+
+	assert.True(t, keyspace["avg_ttl"].(int64) >= 0)
+	assert.True(t, keyspace["expires"].(int64) >= 0)
+	assert.True(t, keyspace["keys"].(int64) >= 0)
+	assert.True(t, strings.Contains(keyspace["id"].(string), "db"))
+}
+
+func testData(t *testing.T, r compose.R) {
+	addEntry(t, r.Host())
+
+	f := mbtest.NewEventsFetcher(t, getConfig(r.Host()))
+
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
+
 // addEntry adds an entry to redis
 func addEntry(t *testing.T, host string) {
 	// Insert at least one event to make sure db exists
diff --git a/metricbeat/module/traefik/health/health_integration_test.go b/metricbeat/module/traefik/health/health_integration_test.go
index c7fcabc4bad9..48f3d4eef96a 100644
--- a/metricbeat/module/traefik/health/health_integration_test.go
+++ b/metricbeat/module/traefik/health/health_integration_test.go
@@ -43,33 +43,37 @@ func makeBadRequest(config map[string]interface{}) error {
 
 func TestHealth(t *testing.T) {
 	mtest.Runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			config := mtest.GetConfig("health", r.Host())
-
-			makeBadRequest(config)
-
-			ms := mbtest.NewReportingMetricSetV2(t, config)
-			reporter := &mbtest.CapturingReporterV2{}
-
-			ms.Fetch(reporter)
-			assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
-
-			event := reporter.GetEvents()[0]
-			assert.NotNil(t, event)
-			t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
-
-			responseCount, _ := event.MetricSetFields.GetValue("response.count")
-			assert.True(t, responseCount.(int64) >= 1)
-
-			badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
-			assert.True(t, badResponseCount.(float64) >= 1)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
-			err := mbtest.WriteEventsReporterV2(ms, t, "")
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
+
+func testFetch(t *testing.T, r compose.R) {
+	config := mtest.GetConfig("health", r.Host())
+
+	makeBadRequest(config)
+
+	ms := mbtest.NewReportingMetricSetV2(t, config)
+	reporter := &mbtest.CapturingReporterV2{}
+
+	ms.Fetch(reporter)
+	assert.Nil(t, reporter.GetErrors(), "Errors while fetching metrics")
+
+	event := reporter.GetEvents()[0]
+	assert.NotNil(t, event)
+	t.Logf("%s/%s event: %+v", ms.Module().Name(), ms.Name(), event)
+
+	responseCount, _ := event.MetricSetFields.GetValue("response.count")
+	assert.True(t, responseCount.(int64) >= 1)
+
+	badResponseCount, _ := event.MetricSetFields.GetValue("response.status_codes.404")
+	assert.True(t, badResponseCount.(float64) >= 1)
+}
+
+func testData(t *testing.T, r compose.R) {
+	ms := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("health", r.Host()))
+	err := mbtest.WriteEventsReporterV2(ms, t, "")
+	if err != nil {
+		t.Fatal("write", err)
+	}
+}
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index b76380c4b95a..67726728d57b 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -28,35 +28,39 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestStatusTCP(t *testing.T) {
+	testStatus(t, "uwsgi_tcp")
+}
+
+func TestStatusHTTP(t *testing.T) {
+	testStatus(t, "uwsgi_http")
+}
+
 func testStatus(t *testing.T, service string) {
 	runner := compose.TestRunner{Service: service}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
-			events, err := f.Fetch()
-			assert.NoError(t, err)
-
-			assert.True(t, len(events) > 0)
-			totals := findItems(events, "total")
-			assert.Equal(t, 1, len(totals))
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
-			err := mbtest.WriteEvents(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
+		"Fetch": testFetch,
+		"Data":  testData,
 	})
 }
 
-func TestStatusTCP(t *testing.T) {
-	testStatus(t, "uwsgi_tcp")
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+	events, err := f.Fetch()
+	assert.NoError(t, err)
+
+	assert.True(t, len(events) > 0)
+	totals := findItems(events, "total")
+	assert.Equal(t, 1, len(totals))
 }
 
-func TestStatusHTTP(t *testing.T) {
-	testStatus(t, "uwsgi_http")
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+	err := mbtest.WriteEvents(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(t *testing.T, service string, host string) map[string]interface{} {
diff --git a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
index d73aa21b5fba..8c3578ef0db3 100644
--- a/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
+++ b/metricbeat/module/zookeeper/mntr/mntr_integration_test.go
@@ -33,39 +33,42 @@ func TestMntr(t *testing.T) {
 	runner := compose.TestRunner{Service: "zookeeper", Parallel: true}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
-			event, err := f.Fetch()
-			if !assert.NoError(t, err) {
-				t.FailNow()
-			}
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
 
-			t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
+}
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	event, err := f.Fetch()
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
 
-			// Check values
-			version := event["version"].(string)
-			avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-			maxLatency := event["latency"].(common.MapStr)["max"].(int64)
-			numAliveConnections := event["num_alive_connections"].(int64)
+	t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event)
 
-			assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
-			assert.True(t, avgLatency >= 0)
-			assert.True(t, maxLatency >= 0)
-			assert.True(t, numAliveConnections > 0)
+	// Check values
+	version := event["version"].(string)
+	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
+	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+	numAliveConnections := event["num_alive_connections"].(int64)
 
-			// Check number of fields. At least 10, depending on environment
-			assert.True(t, len(event) >= 10)
-		},
-		"Data": func(t *testing.T, r compose.R) {
-			f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
+	assert.Equal(t, version, "3.4.8--1, built on 02/06/2016 03:18 GMT")
+	assert.True(t, avgLatency >= 0)
+	assert.True(t, maxLatency >= 0)
+	assert.True(t, numAliveConnections > 0)
 
-			err := mbtest.WriteEvent(f, t)
-			if err != nil {
-				t.Fatal("write", err)
-			}
-		},
-	})
+	// Check number of fields. At least 10, depending on environment
+	assert.True(t, len(event) >= 10)
+}
+
+func testData(t *testing.T, r compose.R) {
+	f := mbtest.NewEventFetcher(t, getConfig(r.Host()))
 
+	err := mbtest.WriteEvent(f, t)
+	if err != nil {
+		t.Fatal("write", err)
+	}
 }
 
 func getConfig(host string) map[string]interface{} {

From 494aa299904102e0a7b7be6718aca17433ee5e7c Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Fri, 23 Nov 2018 17:11:46 +0100
Subject: [PATCH 77/88] Fix tests after refactor

---
 .../stubstatus/stubstatus_integration_test.go |  2 +-
 .../uwsgi/status/status_integration_test.go   | 19 +++++++++++++------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
index 68d998bcde67..7560df4af2a3 100644
--- a/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
+++ b/metricbeat/module/nginx/stubstatus/stubstatus_integration_test.go
@@ -35,7 +35,7 @@ func TestStubstatus(t *testing.T) {
 	}
 
 	runner.Run(t, compose.Suite{
-		"Fetch": tesFetch,
+		"Fetch": testFetch,
 		"Data":  testData,
 	})
 }
diff --git a/metricbeat/module/uwsgi/status/status_integration_test.go b/metricbeat/module/uwsgi/status/status_integration_test.go
index 67726728d57b..b6ad484d7847 100644
--- a/metricbeat/module/uwsgi/status/status_integration_test.go
+++ b/metricbeat/module/uwsgi/status/status_integration_test.go
@@ -29,24 +29,31 @@ import (
 )
 
 func TestStatusTCP(t *testing.T) {
+	t.Parallel()
 	testStatus(t, "uwsgi_tcp")
 }
 
 func TestStatusHTTP(t *testing.T) {
+	t.Parallel()
 	testStatus(t, "uwsgi_http")
 }
 
 func testStatus(t *testing.T, service string) {
 	runner := compose.TestRunner{Service: service}
 
+	test := uwsgiTest{Service: service}
 	runner.Run(t, compose.Suite{
-		"Fetch": testFetch,
-		"Data":  testData,
+		"Fetch": test.Fetch,
+		"Data":  test.Data,
 	})
 }
 
-func testFetch(t *testing.T, r compose.R) {
-	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+type uwsgiTest struct {
+	Service string
+}
+
+func (u *uwsgiTest) Fetch(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host()))
 	events, err := f.Fetch()
 	assert.NoError(t, err)
 
@@ -55,8 +62,8 @@ func testFetch(t *testing.T, r compose.R) {
 	assert.Equal(t, 1, len(totals))
 }
 
-func testData(t *testing.T, r compose.R) {
-	f := mbtest.NewEventsFetcher(t, getConfig(t, service, r.Host()))
+func (u *uwsgiTest) Data(t *testing.T, r compose.R) {
+	f := mbtest.NewEventsFetcher(t, getConfig(t, u.Service, r.Host()))
 	err := mbtest.WriteEvents(f, t)
 	if err != nil {
 		t.Fatal("write", err)

From 1f0bb6f3e643ac6774b84ea6f7efe6f6561ede6f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 18:17:14 +0100
Subject: [PATCH 78/88] Remove leftover comment

---
 libbeat/tests/compose/project.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libbeat/tests/compose/project.go b/libbeat/tests/compose/project.go
index 89249f88d08e..776f3583d7e1 100644
--- a/libbeat/tests/compose/project.go
+++ b/libbeat/tests/compose/project.go
@@ -59,7 +59,6 @@ type Driver interface {
 	Down(ctx context.Context) error
 	Kill(ctx context.Context, signal string, service string) error
 	Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
-	// Containers(ctx context.Context, projectFilter Filter, filter ...string) ([]string, error)
 
 	SetParameters(map[string]string)
 

From 77e24a7ad65bdfccee6176970b0be41fbb928c3a Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 18:58:12 +0100
Subject: [PATCH 79/88] Try to make hound CI happy

---
 libbeat/tests/compose/runner.go               | 27 ++++++++++++++++---
 metricbeat/module/aerospike/mtest/runner.go   |  1 +
 metricbeat/module/apache/mtest/runner.go      |  1 +
 metricbeat/module/kafka/mtest/helpers.go      |  2 ++
 metricbeat/module/kafka/mtest/runner.go       |  1 +
 metricbeat/module/kibana/mtest/testing.go     |  1 +
 metricbeat/module/logstash/testing.go         |  1 +
 metricbeat/module/mongodb/mtest/testing.go    |  2 ++
 metricbeat/module/mysql/mtest/testing.go      |  1 +
 metricbeat/module/php_fpm/mtest/testing.go    |  2 ++
 metricbeat/module/prometheus/mtest/testing.go |  2 ++
 metricbeat/module/rabbitmq/mtest/runner.go    |  1 +
 metricbeat/module/redis/mtest/runner.go       |  1 +
 metricbeat/module/traefik/mtest/runner.go     |  1 +
 14 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/libbeat/tests/compose/runner.go b/libbeat/tests/compose/runner.go
index 233caf675cb1..12206ef5bdfd 100644
--- a/libbeat/tests/compose/runner.go
+++ b/libbeat/tests/compose/runner.go
@@ -34,15 +34,30 @@ func init() {
 	rand.Seed(time.Now().UnixNano())
 }
 
+// TestRunner starts a service with different combinations of options and
+// runs tests on each one of these combinations
 type TestRunner struct {
-	Service  string
-	Options  RunnerOptions
+	// Name of the service managed by this runner
+	Service string
+
+	// Map of options with the list of possible values
+	Options RunnerOptions
+
+	// Set to true if this runner can run in parallel with other runners
 	Parallel bool
-	Timeout  int
+
+	// Timeout to start the managed service
+	Timeout int
 }
 
+// Suite is a set of tests to be run with a TestRunner
+// Each test must be one of:
+// - func(R)
+// - func(*testing.T, R)
+// - func(*testing.T)
 type Suite map[string]interface{}
 
+// RunnerOptions are the possible options of a runner scenario
 type RunnerOptions map[string][]string
 
 func (r *TestRunner) scenarios() []map[string]string {
@@ -107,6 +122,7 @@ func (r *TestRunner) runHostOverride(t *testing.T, tests Suite) bool {
 	return true
 }
 
+// Run runs a tests suite
 func (r *TestRunner) Run(t *testing.T, tests Suite) {
 	t.Helper()
 
@@ -198,26 +214,31 @@ type runnerControl struct {
 	scenario map[string]string
 }
 
+// WithT creates a copy of R with the given T
 func (r *runnerControl) WithT(t *testing.T) R {
 	ctl := *r
 	ctl.T = t
 	return &ctl
 }
 
+// Host returns the host:port the test should use to connect to the service
 func (r *runnerControl) Host() string {
 	return r.host
 }
 
+// Hostname is the address of the host
 func (r *runnerControl) Hostname() string {
 	hostname, _, _ := net.SplitHostPort(r.host)
 	return hostname
 }
 
+// Port is the port of the host
 func (r *runnerControl) Port() string {
 	_, port, _ := net.SplitHostPort(r.host)
 	return port
 }
 
+// Option returns the value of an option for the current scenario
 func (r *runnerControl) Option(key string) string {
 	return r.scenario[key]
 }
diff --git a/metricbeat/module/aerospike/mtest/runner.go b/metricbeat/module/aerospike/mtest/runner.go
index fbd3ef5b56c6..2463ad13c889 100644
--- a/metricbeat/module/aerospike/mtest/runner.go
+++ b/metricbeat/module/aerospike/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for aerospike
 	Runner = compose.TestRunner{
 		Service: "aerospike",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/apache/mtest/runner.go b/metricbeat/module/apache/mtest/runner.go
index b069377ee912..2b86eedb79cf 100644
--- a/metricbeat/module/apache/mtest/runner.go
+++ b/metricbeat/module/apache/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for apache
 	Runner = compose.TestRunner{
 		Service: "apache",
 		Options: map[string][]string{
diff --git a/metricbeat/module/kafka/mtest/helpers.go b/metricbeat/module/kafka/mtest/helpers.go
index 024784a210ec..4f3c309260d2 100644
--- a/metricbeat/module/kafka/mtest/helpers.go
+++ b/metricbeat/module/kafka/mtest/helpers.go
@@ -26,6 +26,7 @@ import (
 	saramacluster "github.com/bsm/sarama-cluster"
 )
 
+// GenerateKafkaData generates kafka data for tests
 func GenerateKafkaData(t *testing.T, topic string, host string) {
 	t.Logf("Send Kafka Event to topic: %v", topic)
 
@@ -64,6 +65,7 @@ func GenerateKafkaData(t *testing.T, topic string, host string) {
 	}
 }
 
+// StartConsumer starts a kafka consumer for tests
 func StartConsumer(t *testing.T, topic, host string) (io.Closer, error) {
 	brokers := []string{host}
 	topics := []string{topic}
diff --git a/metricbeat/module/kafka/mtest/runner.go b/metricbeat/module/kafka/mtest/runner.go
index e3b6a5c2e807..56ce7b056b61 100644
--- a/metricbeat/module/kafka/mtest/runner.go
+++ b/metricbeat/module/kafka/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for kafka tests
 	Runner = compose.TestRunner{
 		Service: "kafka",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/kibana/mtest/testing.go b/metricbeat/module/kibana/mtest/testing.go
index a8f1d191f80d..968a78d60748 100644
--- a/metricbeat/module/kibana/mtest/testing.go
+++ b/metricbeat/module/kibana/mtest/testing.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for kibana tests
 	Runner = compose.TestRunner{
 		Service:  "kibana",
 		Parallel: true,
diff --git a/metricbeat/module/logstash/testing.go b/metricbeat/module/logstash/testing.go
index 3a7137e89875..01055e93181e 100644
--- a/metricbeat/module/logstash/testing.go
+++ b/metricbeat/module/logstash/testing.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is the compose test runner for logstash
 	Runner = compose.TestRunner{
 		Service:  "logstash",
 		Parallel: true,
diff --git a/metricbeat/module/mongodb/mtest/testing.go b/metricbeat/module/mongodb/mtest/testing.go
index 5d4200825a81..f1657d1f161f 100644
--- a/metricbeat/module/mongodb/mtest/testing.go
+++ b/metricbeat/module/mongodb/mtest/testing.go
@@ -21,11 +21,13 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is the compose test runner for mongodb
 var Runner = compose.TestRunner{
 	Service:  "mongodb",
 	Parallel: true,
 }
 
+// GetConfig creates a config for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "mongodb",
diff --git a/metricbeat/module/mysql/mtest/testing.go b/metricbeat/module/mysql/mtest/testing.go
index 9a8410ccb8d9..62477a029e41 100644
--- a/metricbeat/module/mysql/mtest/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -23,6 +23,7 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is a compose test runner for mysql
 var Runner = compose.TestRunner{
 	Service:  "mysql",
 	Parallel: true,
diff --git a/metricbeat/module/php_fpm/mtest/testing.go b/metricbeat/module/php_fpm/mtest/testing.go
index 7f427379fb49..bf5cce996d3e 100644
--- a/metricbeat/module/php_fpm/mtest/testing.go
+++ b/metricbeat/module/php_fpm/mtest/testing.go
@@ -23,11 +23,13 @@ import (
 	"github.com/elastic/beats/libbeat/tests/compose"
 )
 
+// Runner is a compose test runner for php fpm
 var Runner = compose.TestRunner{
 	Service:  "phpfpm",
 	Parallel: true,
 }
 
+// GetConfig creates a configuration for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "php_fpm",
diff --git a/metricbeat/module/prometheus/mtest/testing.go b/metricbeat/module/prometheus/mtest/testing.go
index f6b7f18106f6..fb32c5e7a380 100644
--- a/metricbeat/module/prometheus/mtest/testing.go
+++ b/metricbeat/module/prometheus/mtest/testing.go
@@ -22,12 +22,14 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for prometheus
 	Runner = compose.TestRunner{
 		Service:  "prometheus",
 		Parallel: true,
 	}
 )
 
+// GetConfig creates a configuration for a metricset and host
 func GetConfig(metricset, host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "prometheus",
diff --git a/metricbeat/module/rabbitmq/mtest/runner.go b/metricbeat/module/rabbitmq/mtest/runner.go
index a70257d78b9b..f8b8a0c0a278 100644
--- a/metricbeat/module/rabbitmq/mtest/runner.go
+++ b/metricbeat/module/rabbitmq/mtest/runner.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for RabbitMQ tests
 	Runner = compose.TestRunner{
 		Service:  "rabbitmq",
 		Parallel: true,
diff --git a/metricbeat/module/redis/mtest/runner.go b/metricbeat/module/redis/mtest/runner.go
index 6d3dbd546e83..4ffb857ffc77 100644
--- a/metricbeat/module/redis/mtest/runner.go
+++ b/metricbeat/module/redis/mtest/runner.go
@@ -24,6 +24,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for Redis tests
 	Runner = compose.TestRunner{
 		Service: "redis",
 		Options: compose.RunnerOptions{
diff --git a/metricbeat/module/traefik/mtest/runner.go b/metricbeat/module/traefik/mtest/runner.go
index 10ce0edf1623..896aca2eab7f 100644
--- a/metricbeat/module/traefik/mtest/runner.go
+++ b/metricbeat/module/traefik/mtest/runner.go
@@ -22,6 +22,7 @@ import (
 )
 
 var (
+	// Runner is a compose test runner for traefik
 	Runner = compose.TestRunner{
 		Service:  "traefik",
 		Parallel: true,

From d80c4d4f5dfa00e50ba81e18f51e8b6767420dbd Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 19:01:23 +0100
Subject: [PATCH 80/88] Remove commented out line

---
 libbeat/scripts/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile
index 23e08ce032a7..b07f2f04f372 100755
--- a/libbeat/scripts/Makefile
+++ b/libbeat/scripts/Makefile
@@ -375,7 +375,6 @@ import-dashboards: update ${BEAT_NAME}
 # Builds the environment to test beat
 .PHONY: build-image
 build-image: write-environment
-	# ${DOCKER_COMPOSE} build ${DOCKER_NOCACHE} --pull --force-rm
 
 # Runs the environment so the redis and elasticsearch can also be used for local development
 # To use it for running the test, set ES_HOST and REDIS_HOST environment variable to the ip of your docker-machine.

From fc39b364271925210afb11486923ff1e2f3159c9 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Tue, 27 Nov 2018 19:17:43 +0100
Subject: [PATCH 81/88] Add tests for scenarios

---
 libbeat/tests/compose/runner_test.go | 80 ++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 libbeat/tests/compose/runner_test.go

diff --git a/libbeat/tests/compose/runner_test.go b/libbeat/tests/compose/runner_test.go
new file mode 100644
index 000000000000..30b5d1884607
--- /dev/null
+++ b/libbeat/tests/compose/runner_test.go
@@ -0,0 +1,80 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package compose
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRunnerScenarios(t *testing.T) {
+	cases := []struct {
+		Title    string
+		Options  RunnerOptions
+		Expected []map[string]string
+	}{
+		{
+			Title:    "Nil options",
+			Options:  nil,
+			Expected: []map[string]string{nil},
+		},
+		{
+			Title:    "Empty options",
+			Options:  RunnerOptions{},
+			Expected: []map[string]string{nil},
+		},
+		{
+			Title: "One option, two values",
+			Options: RunnerOptions{
+				"FOO": {"bar", "baz"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar"},
+				{"FOO": "baz"},
+			},
+		},
+		{
+			Title: "Multiple options",
+			Options: RunnerOptions{
+				"FOO": {"bar", "baz"},
+				"BAZ": {"stuff"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar", "BAZ": "stuff"},
+				{"FOO": "baz", "BAZ": "stuff"},
+			},
+		},
+		{
+			Title: "Multiple options, single values",
+			Options: RunnerOptions{
+				"FOO": {"bar"},
+				"BAZ": {"stuff"},
+			},
+			Expected: []map[string]string{
+				{"FOO": "bar", "BAZ": "stuff"},
+			},
+		},
+	}
+
+	for _, c := range cases {
+		r := TestRunner{Options: c.Options}
+		found := r.scenarios()
+		assert.Equal(t, c.Expected, found, c.Title)
+	}
+}

From 0de565e734648d36a33ff228aa50cff432063aba Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 20:59:26 +0100
Subject: [PATCH 82/88] Fix nats tests

---
 metricbeat/docker-compose.yml                 |  2 +
 metricbeat/module/nats/_meta/env              |  2 -
 .../connections_integration_test.go           | 41 ++++++-------------
 metricbeat/module/nats/mtest/runner.go        | 31 ++++++++++++++
 .../nats/routes/routes_integration_test.go    | 41 ++++++-------------
 .../nats/stats/stats_integration_test.go      | 41 ++++++-------------
 metricbeat/tests/system/test_nats.py          |  5 +--
 7 files changed, 73 insertions(+), 90 deletions(-)
 delete mode 100644 metricbeat/module/nats/_meta/env
 create mode 100644 metricbeat/module/nats/mtest/runner.go

diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml
index e8515ffd2818..4e596f55674a 100644
--- a/metricbeat/docker-compose.yml
+++ b/metricbeat/docker-compose.yml
@@ -190,6 +190,8 @@ services:
 
   nats:
     build: ./module/nats/_meta
+    ports:
+      - 8222
 
   nginx:
     build: ./module/nginx/_meta
diff --git a/metricbeat/module/nats/_meta/env b/metricbeat/module/nats/_meta/env
deleted file mode 100644
index 69e439d0a4fd..000000000000
--- a/metricbeat/module/nats/_meta/env
+++ /dev/null
@@ -1,2 +0,0 @@
-NATS_HOST=nats
-NATS_PORT=8222
diff --git a/metricbeat/module/nats/connections/connections_integration_test.go b/metricbeat/module/nats/connections/connections_integration_test.go
index c5bc1bbd0954..c15c36038080 100644
--- a/metricbeat/module/nats/connections/connections_integration_test.go
+++ b/metricbeat/module/nats/connections/connections_integration_test.go
@@ -20,57 +20,42 @@
 package connections
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestConnections(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"connections"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/nats/mtest/runner.go b/metricbeat/module/nats/mtest/runner.go
new file mode 100644
index 000000000000..25174c34361a
--- /dev/null
+++ b/metricbeat/module/nats/mtest/runner.go
@@ -0,0 +1,31 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	// Runner is a compose test runner for Nats tests
+	Runner = compose.TestRunner{
+		Service: "nats",
+	}
+)
diff --git a/metricbeat/module/nats/routes/routes_integration_test.go b/metricbeat/module/nats/routes/routes_integration_test.go
index 8481fc3a49c9..57e1f8213c52 100644
--- a/metricbeat/module/nats/routes/routes_integration_test.go
+++ b/metricbeat/module/nats/routes/routes_integration_test.go
@@ -20,57 +20,42 @@
 package routes
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	mtest "github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestRoutes(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"routes"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/module/nats/stats/stats_integration_test.go b/metricbeat/module/nats/stats/stats_integration_test.go
index 171f7b7904f5..db82d381f191 100644
--- a/metricbeat/module/nats/stats/stats_integration_test.go
+++ b/metricbeat/module/nats/stats/stats_integration_test.go
@@ -20,57 +20,42 @@
 package stats
 
 import (
-	"os"
 	"testing"
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/nats/mtest"
 )
 
-func TestData(t *testing.T) {
-	compose.EnsureUp(t, "nats")
+func TestStats(t *testing.T) {
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+func testData(t *testing.T, r compose.R) {
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	err := mbtest.WriteEventsReporterV2(metricSet, t, "./test_data.json")
 	if err != nil {
 		t.Fatal("write", err)
 	}
 }
 
-func TestFetch(t *testing.T) {
-	compose.EnsureUp(t, "nats")
-
+func testFetch(t *testing.T, r compose.R) {
 	reporter := &mbtest.CapturingReporterV2{}
 
-	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig())
+	metricSet := mbtest.NewReportingMetricSetV2(t, getConfig(r.Host()))
 	metricSet.Fetch(reporter)
 
 	e := mbtest.StandardizeEvent(metricSet, reporter.GetEvents()[0])
 	t.Logf("%s/%s event: %+v", metricSet.Module().Name(), metricSet.Name(), e.Fields.StringToPrint())
 }
 
-func getConfig() map[string]interface{} {
+func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "nats",
 		"metricsets": []string{"stats"},
-		"hosts":      []string{GetEnvHost() + ":" + GetEnvPort()},
-	}
-}
-
-func GetEnvHost() string {
-	host := os.Getenv("NATS_HOST")
-
-	if len(host) == 0 {
-		host = "127.0.0.1"
-	}
-	return host
-}
-
-func GetEnvPort() string {
-	port := os.Getenv("NATS_PORT")
-
-	if len(port) == 0 {
-		port = "8222"
+		"hosts":      []string{host},
 	}
-	return port
 }
diff --git a/metricbeat/tests/system/test_nats.py b/metricbeat/tests/system/test_nats.py
index c40badd42a6d..352f927b05d5 100644
--- a/metricbeat/tests/system/test_nats.py
+++ b/metricbeat/tests/system/test_nats.py
@@ -85,7 +85,4 @@ def test_routes(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return ["{}:{}".format(
-            os.getenv('NATS_HOST', 'localhost'),
-            os.getenv('NATS_PORT', '8222')
-        )]
+        return [self.compose_host()]

From a200df2bfb7d59304f6c8322e278a0d98d166ffa Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 21:13:53 +0100
Subject: [PATCH 83/88] Add ES versions

---
 .../module/elasticsearch/elasticsearch_integration_test.go  | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
index 1ae6a710ab1e..1c30f0c38147 100644
--- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
+++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go
@@ -62,10 +62,12 @@ func TestElasticsearch(t *testing.T) {
 		Service: "elasticsearch",
 		Options: compose.RunnerOptions{
 			"ELASTICSEARCH_VERSION": {
+				// "7.0.0-alpha2",
+				"6.5.4",
 				"6.4.3",
 				"6.3.2",
 				"6.2.4",
-				"5.6.11",
+				"5.6.14",
 			},
 		},
 		Parallel: true,
@@ -78,7 +80,7 @@ func TestElasticsearch(t *testing.T) {
 }
 
 func testFetch(t *testing.T, r compose.R) {
-	if r.Option("ELASTICSEARCH_VERSION") == "6.2.4" {
+	if v := r.Option("ELASTICSEARCH_VERSION"); v == "6.2.4" || v == "5.6.14" {
 		t.Skip("This test fails on this version")
 	}
 

From 76a402c936a77c0b974bc9e7d79d563efd2ee16f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sat, 19 Jan 2019 21:45:11 +0100
Subject: [PATCH 84/88] Fix mssql tests

---
 x-pack/metricbeat/docker-compose.yml           |  5 +++--
 x-pack/metricbeat/module/mssql/_meta/env       |  4 ----
 .../module/mssql/db/data_integration_test.go   |  7 ++++---
 .../module/mssql/db/db_integration_test.go     | 14 ++++++++++----
 .../module/mssql/{testing => mtest}/mssql.go   |  8 ++++----
 x-pack/metricbeat/module/mssql/mtest/runner.go | 18 ++++++++++++++++++
 .../mssql/performance/data_integration_test.go | 18 +++++++-----------
 .../performance_integration_test.go            | 17 +++++++++++------
 x-pack/metricbeat/tests/system/test_mssql.py   |  2 +-
 9 files changed, 58 insertions(+), 35 deletions(-)
 delete mode 100644 x-pack/metricbeat/module/mssql/_meta/env
 rename x-pack/metricbeat/module/mssql/{testing => mtest}/mssql.go (79%)
 create mode 100644 x-pack/metricbeat/module/mssql/mtest/runner.go

diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml
index 1a67babb75d1..5d3bb202f239 100644
--- a/x-pack/metricbeat/docker-compose.yml
+++ b/x-pack/metricbeat/docker-compose.yml
@@ -8,10 +8,11 @@ services:
     volumes:
       - ${PWD}/../..:/go/src/github.com/elastic/beats/
       - /var/run/docker.sock:/var/run/docker.sock
+    network_mode: host
     command: make
-    env_file:
-      - ./module/mssql/_meta/env
 
   # Modules
   mssql:
     build: ./module/mssql/_meta
+    ports:
+      - 1433
diff --git a/x-pack/metricbeat/module/mssql/_meta/env b/x-pack/metricbeat/module/mssql/_meta/env
deleted file mode 100644
index 145a143e51f8..000000000000
--- a/x-pack/metricbeat/module/mssql/_meta/env
+++ /dev/null
@@ -1,4 +0,0 @@
-MSSQL_HOST=mssql
-MSSQL_PORT=1433
-MSSQL_USER=SA
-MSSQL_PASSWORD=1234_asdf
diff --git a/x-pack/metricbeat/module/mssql/db/data_integration_test.go b/x-pack/metricbeat/module/mssql/db/data_integration_test.go
index 7a8fc8bafc5d..a64dcb41f651 100644
--- a/x-pack/metricbeat/module/mssql/db/data_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/db/data_integration_test.go
@@ -7,14 +7,15 @@ package db
 import (
 	"testing"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestData(t *testing.T) {
+func testData(t *testing.T, r compose.R) {
 	t.Skip("Skipping `data.json` generation test")
 
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db"))
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "db"))
 
 	err := mbtest.WriteEventsReporterV2(f, t, "")
 	if err != nil {
diff --git a/x-pack/metricbeat/module/mssql/db/db_integration_test.go b/x-pack/metricbeat/module/mssql/db/db_integration_test.go
index 4fc9f594b7da..da531e5ac599 100644
--- a/x-pack/metricbeat/module/mssql/db/db_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/db/db_integration_test.go
@@ -14,14 +14,20 @@ import (
 	"github.com/elastic/beats/libbeat/logp"
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestFetch(t *testing.T) {
+func TestDb(t *testing.T) {
 	logp.TestingSetup()
-	compose.EnsureUp(t, "mssql")
 
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("db"))
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+		"Data":  testData,
+	})
+}
+
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "db"))
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
diff --git a/x-pack/metricbeat/module/mssql/testing/mssql.go b/x-pack/metricbeat/module/mssql/mtest/mssql.go
similarity index 79%
rename from x-pack/metricbeat/module/mssql/testing/mssql.go
rename to x-pack/metricbeat/module/mssql/mtest/mssql.go
index 971b06badb5c..7d405af6ed06 100644
--- a/x-pack/metricbeat/module/mssql/testing/mssql.go
+++ b/x-pack/metricbeat/module/mssql/mtest/mssql.go
@@ -2,19 +2,19 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package testing
+package mtest
 
 import "os"
 
 // GetConfig returns the required configuration options for testing a MSSQL
 // metricset.
-func GetConfig(metricSets ...string) map[string]interface{} {
+func GetConfig(host string, metricSets ...string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "mssql",
 		"metricsets": metricSets,
-		"hosts":      []string{EnvOr("MSSQL_HOST", "localhost")},
+		"hosts":      []string{host},
 		"username":   EnvOr("MSSQL_USER", "SA"),
-		"password":   EnvOr("MSSQL_PASSWORD", ""),
+		"password":   EnvOr("MSSQL_PASSWORD", "1234_asdf"),
 	}
 }
 
diff --git a/x-pack/metricbeat/module/mssql/mtest/runner.go b/x-pack/metricbeat/module/mssql/mtest/runner.go
new file mode 100644
index 000000000000..1a8c2b8033fc
--- /dev/null
+++ b/x-pack/metricbeat/module/mssql/mtest/runner.go
@@ -0,0 +1,18 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+// +build integration
+
+package mtest
+
+import (
+	"github.com/elastic/beats/libbeat/tests/compose"
+)
+
+var (
+	// Runner is a compose test runner for Redis tests
+	Runner = compose.TestRunner{
+		Service: "mssql",
+	}
+)
diff --git a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
index 9c6885937838..f93e051cadf1 100644
--- a/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/performance/data_integration_test.go
@@ -12,13 +12,14 @@ import (
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
-func TestData(t *testing.T) {
+func testData(t *testing.T, r compose.R) {
 	t.Skip("Skipping `data.json` generation test")
-	_, config, err := getHostURI()
+	_, config, err := getHostURI(r.Host())
 	if err != nil {
 		t.Fatal("error getting config information", err.Error())
 	}
@@ -35,13 +36,8 @@ func TestData(t *testing.T) {
 	}
 }
 
-func getHostURI() (string, map[string]interface{}, error) {
-	config := mtest.GetConfig("performance")
-
-	host, ok := config["hosts"].([]string)
-	if !ok {
-		return "", nil, errors.New("error getting host name information")
-	}
+func getHostURI(host string) (string, map[string]interface{}, error) {
+	config := mtest.GetConfig(host, "performance")
 
 	username, ok := config["username"].(string)
 	if !ok {
@@ -56,7 +52,7 @@ func getHostURI() (string, map[string]interface{}, error) {
 	u := &url.URL{
 		Scheme: "sqlserver",
 		User:   url.UserPassword(username, password),
-		Host:   host[0],
+		Host:   host,
 	}
 
 	return u.String(), config, nil
diff --git a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
index 6aae33523ee8..a6cb11a9ec35 100644
--- a/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
+++ b/x-pack/metricbeat/module/mssql/performance/performance_integration_test.go
@@ -15,19 +15,24 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	mtest "github.com/elastic/beats/x-pack/metricbeat/module/mssql/testing"
+	"github.com/elastic/beats/x-pack/metricbeat/module/mssql/mtest"
 )
 
+func TestPerformance(t *testing.T) {
+	logp.TestingSetup()
+
+	mtest.Runner.Run(t, compose.Suite{
+		"Fetch": testFetch,
+	})
+}
+
 type keyAssertion struct {
 	key       string
 	assertion func(v interface{}, key string)
 }
 
-func TestFetch(t *testing.T) {
-	logp.TestingSetup()
-	compose.EnsureUp(t, "mssql")
-
-	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig("performance"))
+func testFetch(t *testing.T, r compose.R) {
+	f := mbtest.NewReportingMetricSetV2(t, mtest.GetConfig(r.Host(), "performance"))
 	events, errs := mbtest.ReportingFetchV2(f)
 	if len(errs) > 0 {
 		t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs)
diff --git a/x-pack/metricbeat/tests/system/test_mssql.py b/x-pack/metricbeat/tests/system/test_mssql.py
index 252c8a1cb066..885f2e5e384f 100644
--- a/x-pack/metricbeat/tests/system/test_mssql.py
+++ b/x-pack/metricbeat/tests/system/test_mssql.py
@@ -75,7 +75,7 @@ def test_performance(self):
         self.assert_fields_are_documented(evt)
 
     def get_hosts(self):
-        return [os.getenv('MSSQL_HOST', 'mssql')]
+        return [self.compose_host()]
 
     def get_username(self):
         return os.getenv('MSSQL_USERNAME', 'SA')

From fae8cda20efac3eac4476d5ecb36ad09bf1a827f Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 20 Jan 2019 02:15:55 +0100
Subject: [PATCH 85/88] Update versions of the stack

---
 metricbeat/module/elasticsearch/docker-compose.yml | 2 +-
 metricbeat/module/kibana/_meta/Dockerfile          | 2 +-
 metricbeat/module/logstash/_meta/Dockerfile        | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/metricbeat/module/elasticsearch/docker-compose.yml b/metricbeat/module/elasticsearch/docker-compose.yml
index e046f6eefed3..518bb6bf8532 100644
--- a/metricbeat/module/elasticsearch/docker-compose.yml
+++ b/metricbeat/module/elasticsearch/docker-compose.yml
@@ -2,7 +2,7 @@ version: "2.1"
 
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.4.3}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-6.5.4}
     environment:
       - "ES_JAVA_OPTS=-Xms90m -Xmx90m"
       - "network.host="
diff --git a/metricbeat/module/kibana/_meta/Dockerfile b/metricbeat/module/kibana/_meta/Dockerfile
index ce77bb67674e..a6cef013f57a 100644
--- a/metricbeat/module/kibana/_meta/Dockerfile
+++ b/metricbeat/module/kibana/_meta/Dockerfile
@@ -1,2 +1,2 @@
-FROM docker.elastic.co/kibana/kibana:6.5.1
+FROM docker.elastic.co/kibana/kibana:6.5.4
 HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:5601/api/status | grep '"disconnects"'
diff --git a/metricbeat/module/logstash/_meta/Dockerfile b/metricbeat/module/logstash/_meta/Dockerfile
index 4c47a068ed15..7ebabe0dc446 100644
--- a/metricbeat/module/logstash/_meta/Dockerfile
+++ b/metricbeat/module/logstash/_meta/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/logstash/logstash:6.3.0
+FROM docker.elastic.co/logstash/logstash:6.5.4
 
 COPY healthcheck.sh /
 ENV XPACK_MONITORING_ENABLED=FALSE

From ffe2062475145f8a381cc9daaa6ef1cde7daf160 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Sun, 20 Jan 2019 12:46:41 +0100
Subject: [PATCH 86/88] Fix redis key test

---
 metricbeat/tests/system/test_redis.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/metricbeat/tests/system/test_redis.py b/metricbeat/tests/system/test_redis.py
index c4793906a67b..a3ca518e1679 100644
--- a/metricbeat/tests/system/test_redis.py
+++ b/metricbeat/tests/system/test_redis.py
@@ -95,9 +95,10 @@ def test_key(self):
         """
 
         # At least one event must be inserted so db stats exist
+        host, port = self.compose_host().split(":")
         r = redis.StrictRedis(
-            host=self.compose_hosts()[0],
-            port=os.getenv('REDIS_PORT', '6379'),
+            host=host,
+            port=port,
             db=0)
         r.flushall()
         r.rpush('list-key', 'one', 'two', 'three')

From 64dfd65b9f987edc58e29fff9c2a125251486879 Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 6 Mar 2019 18:34:56 +0100
Subject: [PATCH 87/88] Move couchbase DSN helper to mtest

---
 .../module/couchbase/bucket/bucket_integration_test.go       | 4 ++--
 .../module/couchbase/cluster/cluster_integration_test.go     | 4 ++--
 metricbeat/module/couchbase/{testing.go => mtest/dsn.go}     | 5 +++--
 metricbeat/module/couchbase/node/node_integration_test.go    | 4 ++--
 4 files changed, 9 insertions(+), 8 deletions(-)
 rename metricbeat/module/couchbase/{testing.go => mtest/dsn.go} (90%)

diff --git a/metricbeat/module/couchbase/bucket/bucket_integration_test.go b/metricbeat/module/couchbase/bucket/bucket_integration_test.go
index 302d1327daf7..4cbe93210e9b 100644
--- a/metricbeat/module/couchbase/bucket/bucket_integration_test.go
+++ b/metricbeat/module/couchbase/bucket/bucket_integration_test.go
@@ -24,7 +24,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestBucket(t *testing.T) {
@@ -46,6 +46,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"bucket"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }
diff --git a/metricbeat/module/couchbase/cluster/cluster_integration_test.go b/metricbeat/module/couchbase/cluster/cluster_integration_test.go
index c67e3ff51450..949dc3b4f059 100644
--- a/metricbeat/module/couchbase/cluster/cluster_integration_test.go
+++ b/metricbeat/module/couchbase/cluster/cluster_integration_test.go
@@ -26,7 +26,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestNode(t *testing.T) {
@@ -59,6 +59,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"cluster"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }
diff --git a/metricbeat/module/couchbase/testing.go b/metricbeat/module/couchbase/mtest/dsn.go
similarity index 90%
rename from metricbeat/module/couchbase/testing.go
rename to metricbeat/module/couchbase/mtest/dsn.go
index bb52baee9251..993d17c8bd8c 100644
--- a/metricbeat/module/couchbase/testing.go
+++ b/metricbeat/module/couchbase/mtest/dsn.go
@@ -15,8 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package couchbase
+package mtest
 
-func GetEnvDSN(host string) string {
+// GetDSN gets the test DSN for a given host
+func GetDSN(host string) string {
 	return "http://Administrator:password@" + host
 }
diff --git a/metricbeat/module/couchbase/node/node_integration_test.go b/metricbeat/module/couchbase/node/node_integration_test.go
index 82b587f53200..8072131a24c6 100644
--- a/metricbeat/module/couchbase/node/node_integration_test.go
+++ b/metricbeat/module/couchbase/node/node_integration_test.go
@@ -26,7 +26,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/tests/compose"
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
-	"github.com/elastic/beats/metricbeat/module/couchbase"
+	"github.com/elastic/beats/metricbeat/module/couchbase/mtest"
 )
 
 func TestNode(t *testing.T) {
@@ -59,6 +59,6 @@ func getConfig(host string) map[string]interface{} {
 	return map[string]interface{}{
 		"module":     "couchbase",
 		"metricsets": []string{"node"},
-		"hosts":      []string{couchbase.GetEnvDSN(host)},
+		"hosts":      []string{mtest.GetDSN(host)},
 	}
 }

From 2c9cd9757f8f7a001cbf8b43b437cbb436539edc Mon Sep 17 00:00:00 2001
From: Jaime Soriano Pastor <jaime.soriano@elastic.co>
Date: Wed, 6 Mar 2019 18:56:32 +0100
Subject: [PATCH 88/88] Fix mysql tests

---
 metricbeat/module/mysql/mtest/testing.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/metricbeat/module/mysql/mtest/testing.go b/metricbeat/module/mysql/mtest/testing.go
index 62477a029e41..1e6f585b1de6 100644
--- a/metricbeat/module/mysql/mtest/testing.go
+++ b/metricbeat/module/mysql/mtest/testing.go
@@ -36,6 +36,10 @@ func GetDSN(host string) string {
 		Addr:   host,
 		User:   "root",
 		Passwd: "test",
+
+		// Required if password is set and FormatDSN() is used
+		// since clients for MySQL 8.0
+		AllowNativePasswords: true,
 	}
 	return c.FormatDSN()
 }