Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
0cfe2c6
Add compose test runner
jsoriano Jul 23, 2019
61ef8af
Migrate apache module to new tests
jsoriano Jul 23, 2019
a7571c5
Don't recreate on EnsureUp
jsoriano Jul 23, 2019
cbd580b
Add flags for reuse and cleanup
jsoriano Jul 23, 2019
6a3972a
Expose host in EnsureUp
jsoriano Jul 24, 2019
9039a75
Redis module
jsoriano Jul 24, 2019
e20f5df
Remove env files
jsoriano Jul 24, 2019
256d94c
Remove all usages of environment variables for hosts
jsoriano Jul 24, 2019
c5e3f8b
Revert changes for test runner
jsoriano Jul 24, 2019
f32f91a
Add comment on setup advertised host
jsoriano Jul 24, 2019
c78cdc5
Remove more unused code
jsoriano Jul 24, 2019
91600f1
Revert docker compose changes for apache
jsoriano Jul 24, 2019
c258f73
Give more time to KillOld
jsoriano Jul 25, 2019
3947210
Add advertised address to containers in python tests
jsoriano Jul 26, 2019
ba8680f
Increase timeout in rabbitmq test
jsoriano Jul 26, 2019
a07dbc1
Allow selection of port for containers with multiple ports
jsoriano Jul 26, 2019
c9e9d0b
Increase timeout for elasticsearch
jsoriano Jul 26, 2019
576c0e6
Copy advertised host env with absolute path
jsoriano Jul 26, 2019
1da2b39
Copy advertised host environment only when requested
jsoriano Jul 26, 2019
4597193
Run ceph integration tests only on linux
jsoriano Jul 29, 2019
f40f1a6
Refactor NO_COMPOSE envs handling
jsoriano Jul 29, 2019
8a80ccf
Keep container status interface
jsoriano Jul 29, 2019
f4e4f1a
Remove unneeded comment
jsoriano Jul 29, 2019
e1f60ac
Fix comments on NO_COMPOSE
jsoriano Jul 29, 2019
c774eb3
Reorder interface
jsoriano Jul 29, 2019
fc04ade
Remove runner as it is not used
jsoriano Jul 29, 2019
b0b7468
Minor fix for old checker
jsoriano Jul 29, 2019
2b6905b
Don't try to kill non-running containers
jsoriano Jul 29, 2019
f30d1ac
Reuse client connections and skip last sleep in wait
jsoriano Jul 29, 2019
5147b0a
Unexport wrapper driver builder
jsoriano Jul 29, 2019
4f8b471
Tune some timeouts
jsoriano Jul 30, 2019
9dcb2f9
Remove unneeded retries in compose up
jsoriano Jul 31, 2019
0e080c4
Remove more unrelated changes
jsoriano Jul 31, 2019
e8941df
Remove unneeded code from compose.py
jsoriano Jul 31, 2019
b523145
Keep skip in rabbitmq flaky test
jsoriano Jul 31, 2019
aa1b613
Rename r to service
jsoriano Aug 2, 2019
71eb5b8
Add documentation to HostInfo methods
jsoriano Aug 2, 2019
591057a
Add options to ensure up
jsoriano Aug 2, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 105 additions & 27 deletions libbeat/tests/compose/compose.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,74 +19,152 @@ package compose

import (
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
)

// HostInfo exposes information about started scenario
type HostInfo interface {
// Host returns an address as host:port that can be used to connect to
// a running service.
Host() string
Comment thread
jsoriano marked this conversation as resolved.

// HostForPort returns an address as host:port that can be used to
// connect to a running service that has multiple exposed ports. The
// address returned is the one that can be used to connect to the
// indicated exposed port.
HostForPort(port int) string
}

// EnsureUp starts all the requested services (must be defined in docker-compose.yml)
// with a default timeout of 300 seconds
func EnsureUp(t *testing.T, services ...string) {
EnsureUpWithTimeout(t, 60, services...)
}
func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo {
t.Helper()

// EnsureUpWithTimeout starts all the requested services (must be defined in docker-compose.yml)
// Wait for `timeout` seconds for health
func EnsureUpWithTimeout(t *testing.T, timeout int, services ...string) {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what’s the reason to accept only one service? not saying it’s a bad idea, just want to know the reasoning

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are two main reasons:

  • It simplifies the returned information. With an only service, only an object with the host information for a service is returned.
  • It doesn't seem to be needed, we only had an scenario with two services (Kibana, that also needs Elasticsearch), and it can also be done with docker compose dependencies. If access to both hosts were needed, we could have two EnsureUp.

If wouldn't be hard to implement it though, by returning a struct with all the services info and adding a method like HostForServicePort(service, port) HostInfo. But I didn't consider it neccessary by now.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That sounds good to me, thanks for the explanation!

// The NO_COMPOSE env variables makes it possible to skip the starting of the environment.
// This is useful if the service is already running locally.
if noCompose, err := strconv.ParseBool(os.Getenv("NO_COMPOSE")); err == nil && noCompose {
return
if hostInfo := HostInfoFromEnv(t, service); hostInfo != nil {
return hostInfo
}

compose, err := getComposeProject()
compose, err := getComposeProject(os.Getenv("DOCKER_COMPOSE_PROJECT_NAME"))
if err != nil {
t.Fatal(err)
}
defer compose.Close()

// Kill no longer used containers
err = compose.KillOld(services)
err = compose.KillOld([]string{service})
if err != nil {
t.Fatal(err)
}

for _, service := range services {
err = compose.Start(service)
if err != nil {
t.Fatal("failed to start service", service, err)
}
upOptions := UpOptions{
Timeout: 60 * time.Second,
Create: CreateOptions{
Build: true,
ForceRecreate: true,
},
}
for _, option := range options {
option(&upOptions)
}

// Start container
err = compose.Start(service, upOptions)
if err != nil {
t.Fatal("failed to start service", service, err)
}

// Wait for health
err = compose.Wait(timeout, services...)
err = compose.Wait(upOptions.Timeout, service)
if err != nil {
t.Fatal(err)
}

// Get host information
host, err := compose.HostInformation(service)
if err != nil {
t.Fatalf("getting host for %s", service)
}

return host
}

func getComposeProject() (*Project, error) {
// EnsureUpWithTimeout starts all the requested services (must be defined in docker-compose.yml)
// Wait for `timeout` seconds for health
func EnsureUpWithTimeout(t testing.TB, timeout int, service string) HostInfo {
return EnsureUp(t, service, UpWithTimeout(time.Duration(timeout)*time.Second))
}

// HostInfoFromEnv gets the host information to use for the test from environment variables.
func HostInfoFromEnv(t testing.TB, service string) HostInfo {
// If an environment variable with the form <SERVICE>_HOST is used, its value
// is used as host instead of starting a new service.
envVar := fmt.Sprintf("%s_HOST", strings.ToUpper(service))
host := os.Getenv(envVar)
if host != "" {
return &staticHostInfo{host: host}
}

// The NO_COMPOSE env variables makes it possible to skip the starting of the environment.
// This is useful if the service is already running locally.
// Kept for historical reasons, now it only complains if the host environment
// variable is not set.
noCompose, err := strconv.ParseBool(os.Getenv("NO_COMPOSE"))
if err == nil && noCompose {
t.Fatalf("%s environment variable must be set as the host:port where %s is running", envVar, service)
}

return nil
}

type staticHostInfo struct {
host string
}

func (i *staticHostInfo) Host() string {
return i.host
}

func (i *staticHostInfo) HostForPort(int) string {
return i.host
}

func findComposePath() (string, error) {
// find docker-compose
path, err := os.Getwd()
if err != nil {
return nil, err
return "", err
}
for {
if path == "/" {
return nil, errors.New("docker-compose.yml not found")
break
}

if _, err = os.Stat(filepath.Join(path, "docker-compose.yml")); err != nil {
path = filepath.Dir(path)
} else {
break
composePath := filepath.Join(path, "docker-compose.yml")
if _, err = os.Stat(composePath); err == nil {
return composePath, nil
}
path = filepath.Dir(path)
}

return "", errors.New("docker-compose.yml not found")
}

func getComposeProject(name string) (*Project, error) {
path, err := findComposePath()
if err != nil {
return nil, err
}

return NewProject(
os.Getenv("DOCKER_COMPOSE_PROJECT_NAME"),
name,
[]string{
filepath.Join(path, "docker-compose.yml"),
path,
},
)
}
Loading