Skip to content

Commit

Permalink
Create a Go script to run Dgraph tests concurrently (#6895)
Browse files Browse the repository at this point in the history 
The current test.sh script runs all the packages one by one, which takes 30mins for the tests to run. Even then, we have to separate out the systests from the rest.

This PR creates an equivalent Go script which can divide up the packages in multiple goroutines. Each goroutine runs a Dgraph cluster which the tests (running on that goroutine) use. To avoid interactions among clusters, the script changes docker-compose to not expose the ports. Instead, the docker generated public ports are used to interact with the cluster. The script makes the cluster prefix available via environment variable, which gets picked up by testutil package to find the right address for the Dgraph instance via Docker.

TODO: This PR disables a bunch of tests and packages which are failing. For now, we set concurrency to 1, due to some weird failures with tests when run with multiple clusters. Needs more work to fix up these failures.

Co-authored-by: Daniel Mai <[email protected]>
  • Loading branch information
@manishrjain @danielmai
manishrjain and danielmai authored Nov 18, 2020
1 parent 17a3dbb commit 79d7f9c
Show file tree
Hide file tree
Showing 61 changed files with 1,848 additions and 1,104 deletions.
120 changes: 114 additions & 6 deletions compose/compose.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"io/ioutil"
"os"
"os/user"
"strconv"
"strings"

sv "github.com/Masterminds/semver/v3"
Expand Down Expand Up @@ -62,6 +63,7 @@ type service struct {
WorkingDir string `yaml:"working_dir,omitempty"`
DependsOn []string `yaml:"depends_on,omitempty"`
Labels stringMap `yaml:",omitempty"`
EnvFile []string `yaml:"env_file,omitempty"`
Environment []string `yaml:",omitempty"`
Ports []string `yaml:",omitempty"`
Volumes []volume `yaml:",omitempty"`
Expand Down Expand Up @@ -106,6 +108,13 @@ type options struct {
LudicrousMode bool
SnapshotAfter string
ContainerNames bool
AlphaVolumes []string
ZeroVolumes []string
AlphaEnvFile []string
ZeroEnvFile []string
Minio bool
MinioPort uint16
MinioEnvFile []string

// Extra flags
AlphaFlags string
Expand Down Expand Up @@ -138,6 +147,9 @@ func toPort(i int) string {
}

func getOffset(idx int) int {
if !opts.ExposePorts {
return 0
}
if idx == 1 {
return 0
}
Expand Down Expand Up @@ -216,7 +228,11 @@ func getZero(idx int) service {
svc.TmpFS = append(svc.TmpFS, fmt.Sprintf("/data/%s/zw", svc.name))
}

svc.Command += fmt.Sprintf(" -o %d --idx=%d", opts.PortOffset+getOffset(idx), idx)
offset := getOffset(idx)
if (opts.PortOffset + offset) != 0 {
svc.Command += fmt.Sprintf(" -o %d", opts.PortOffset+offset)
}
svc.Command += fmt.Sprintf(" --idx=%d", idx)
svc.Command += fmt.Sprintf(" --my=%s:%d", svc.name, grpcPort)
if opts.NumAlphas > 1 {
svc.Command += fmt.Sprintf(" --replicas=%d", opts.NumReplicas)
Expand All @@ -238,6 +254,14 @@ func getZero(idx int) service {
if opts.ZeroFlags != "" {
svc.Command += " " + opts.ZeroFlags
}

if len(opts.ZeroVolumes) > 0 {
for _, vol := range opts.ZeroVolumes {
svc.Volumes = append(svc.Volumes, getVolume(vol))
}
}
svc.EnvFile = opts.ZeroEnvFile

return svc
}

Expand All @@ -256,7 +280,9 @@ func getAlpha(idx int) service {
isMultiZeros := true
var isInvalidVersion, err = semverCompare("< 1.2.3 || 20.03.0", opts.Tag)
if err != nil || isInvalidVersion {
isMultiZeros = false
if opts.Tag != "latest" {
isMultiZeros = false
}
}

maxZeros := 1
Expand All @@ -267,13 +293,16 @@ func getAlpha(idx int) service {
zeroHostAddr := fmt.Sprintf("zero%d:%d", 1, zeroBasePort+opts.PortOffset)
zeros := []string{zeroHostAddr}
for i := 2; i <= maxZeros; i++ {
zeroHostAddr = fmt.Sprintf("zero%d:%d", i, zeroBasePort+opts.PortOffset+i)
zeroHostAddr = fmt.Sprintf("zero%d:%d", i, zeroBasePort+opts.PortOffset+getOffset(i))
zeros = append(zeros, zeroHostAddr)
}

zerosOpt := strings.Join(zeros, ",")

svc.Command += fmt.Sprintf(" -o %d", opts.PortOffset+getOffset(idx))
offset := getOffset(idx)
if (opts.PortOffset + offset) != 0 {
svc.Command += fmt.Sprintf(" -o %d", opts.PortOffset+offset)
}
svc.Command += fmt.Sprintf(" --my=%s:%d", svc.name, internalPort)
svc.Command += fmt.Sprintf(" --zero=%s", zerosOpt)
svc.Command += fmt.Sprintf(" --logtostderr -v=%d", opts.Verbosity)
Expand All @@ -290,7 +319,7 @@ func getAlpha(idx int) service {
svc.Command += " --whitelist=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"
}
if opts.Acl {
svc.Command += " --acl_secret_file=/secret/hmac --acl_access_ttl 3s"
svc.Command += " --acl_secret_file=/secret/hmac"
svc.Volumes = append(svc.Volumes, volume{
Type: "bind",
Source: "./acl-secret",
Expand All @@ -302,7 +331,7 @@ func getAlpha(idx int) service {
svc.Command += fmt.Sprintf(" --snapshot_after=%s", opts.SnapshotAfter)
}
if opts.AclSecret != "" {
svc.Command += " --acl_secret_file=/secret/hmac --acl_access_ttl 3s"
svc.Command += " --acl_secret_file=/secret/hmac"
svc.Volumes = append(svc.Volumes, volume{
Type: "bind",
Source: opts.AclSecret,
Expand Down Expand Up @@ -333,13 +362,37 @@ func getAlpha(idx int) service {
ReadOnly: true,
})
}
if len(opts.AlphaVolumes) > 0 {
for _, vol := range opts.AlphaVolumes {
svc.Volumes = append(svc.Volumes, getVolume(vol))
}
}
svc.EnvFile = opts.AlphaEnvFile
if opts.AlphaFlags != "" {
svc.Command += " " + opts.AlphaFlags
}

return svc
}

func getVolume(vol string) volume {
s := strings.Split(vol, ":")
srcDir := s[0]
dstDir := s[1]
readOnly := len(s) > 2 && s[2] == "ro"
volType := "volume"
if isBindMount(srcDir) {
volType = "bind"
}
return volume{
Type: volType,
Source: srcDir,
Target: dstDir,
ReadOnly: readOnly,
}

}

func getJaeger() service {
svc := service{
Image: "jaegertracing/all-in-one:1.18",
Expand All @@ -359,6 +412,20 @@ func getJaeger() service {
return svc
}

func getMinio() service {
svc := service{
Image: "minio/minio:RELEASE.2020-11-13T20-10-18Z",
ContainerName: containerName("minio1"),
Ports: []string{
toPort(int(opts.MinioPort)),
},
EnvFile: opts.MinioEnvFile,
Command: "minio server /data/minio --address :" +
strconv.FormatUint(uint64(opts.MinioPort), 10),
}
return svc
}

func getRatel() service {
portFlag := ""
if opts.RatelPort != 8000 {
Expand Down Expand Up @@ -448,6 +515,10 @@ func semverCompare(constraint, version string) (bool, error) {
return c.Check(v), nil
}

func isBindMount(vol string) bool {
return strings.HasPrefix(vol, ".") || strings.HasPrefix(vol, "/")
}

func fatal(err error) {
fmt.Fprintf(os.Stderr, "compose: %v\n", err)
os.Exit(1)
Expand Down Expand Up @@ -523,6 +594,20 @@ func main() {
"extra flags for zeros.")
cmd.PersistentFlags().BoolVar(&opts.ContainerNames, "names", true,
"set container names in docker compose.")
cmd.PersistentFlags().StringArrayVar(&opts.AlphaVolumes, "alpha_volume", nil,
"alpha volume mounts, following srcdir:dstdir[:ro]")
cmd.PersistentFlags().StringArrayVar(&opts.ZeroVolumes, "zero_volume", nil,
"zero volume mounts, following srcdir:dstdir[:ro]")
cmd.PersistentFlags().StringArrayVar(&opts.AlphaEnvFile, "alpha_env_file", nil,
"env_file for alpha")
cmd.PersistentFlags().StringArrayVar(&opts.ZeroEnvFile, "zero_env_file", nil,
"env_file for zero")
cmd.PersistentFlags().BoolVar(&opts.Minio, "minio", false,
"include minio service")
cmd.PersistentFlags().Uint16Var(&opts.MinioPort, "minio_port", 9001,
"minio service port")
cmd.PersistentFlags().StringArrayVar(&opts.MinioEnvFile, "minio_env_file", nil,
"minio service env_file")
err := cmd.ParseFlags(os.Args)
if err != nil {
if err == pflag.ErrHelp {
Expand Down Expand Up @@ -570,6 +655,25 @@ func main() {
Volumes: make(map[string]stringMap),
}

if len(opts.AlphaVolumes) > 0 {
for _, vol := range opts.AlphaVolumes {
s := strings.Split(vol, ":")
srcDir := s[0]
if !isBindMount(srcDir) {
cfg.Volumes[srcDir] = stringMap{}
}
}
}
if len(opts.ZeroVolumes) > 0 {
for _, vol := range opts.ZeroVolumes {
s := strings.Split(vol, ":")
srcDir := s[0]
if !isBindMount(srcDir) {
cfg.Volumes[srcDir] = stringMap{}
}
}
}

if opts.DataVol {
cfg.Volumes["data"] = stringMap{}
}
Expand All @@ -586,6 +690,10 @@ func main() {
addMetrics(&cfg)
}

if opts.Minio {
services["minio1"] = getMinio()
}

if opts.Acl {
err = ioutil.WriteFile("acl-secret", []byte("12345678901234567890123456789012"), 0644)
x.Check2(fmt.Fprintf(os.Stdout, "Writing file: %s\n", "acl-secret"))
Expand Down
4 changes: 1 addition & 3 deletions contrib/integration/testtxn/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,9 +407,7 @@ func TestIgnoreIndexConflict(t *testing.T) {
txn = s.dg.NewTxn()
q := `{ me(func: eq(name, "Manish")) { uid }}`
resp, err := txn.Query(context.Background(), q)
if err != nil {
log.Fatalf("Error while running query: %v\n", err)
}
require.NoError(t, err)
expectedResp := []byte(fmt.Sprintf(`{"me":[{"uid":"%s"},{"uid":"%s"}]}`, uid1, uid2))
require.Equal(t, expectedResp, resp.Json)
}
Expand Down
Loading

0 comments on commit 79d7f9c

Please sign in to comment.