better cancel test for hpc compute backends

ohsu-comp-bio · Feb 12, 2018 · e55c5af · e55c5af
1 parent 0de7f8b
commit e55c5af
Show file tree

Hide file tree

Showing 5 changed files with 55 additions and 19 deletions.
diff --git a/tests/funnel_utils.go b/tests/funnel_utils.go
@@ -322,7 +322,7 @@ func (f *Funnel) ReadFile(name string) string {
 }
 
 // StartServerInDocker starts a funnel server in a docker image
-func (f *Funnel) StartServerInDocker(imageName string, extraArgs []string) {
+func (f *Funnel) StartServerInDocker(containerName, imageName string, extraArgs []string) {
 	var funnelBinary string
 	var err error
 	gopath := os.Getenv("GOPATH")
@@ -360,7 +360,7 @@ func (f *Funnel) StartServerInDocker(imageName string, extraArgs []string) {
 	args := []string{
 		"run", "-i", "--rm",
 		"--group-add", fmt.Sprintf("%d", gid),
-		"--name", "funnel-test-server-" + RandomString(6),
+		"--name", containerName,
 		"-p", fmt.Sprintf("%d:%d", httpPort, httpPort),
 		"-p", fmt.Sprintf("%d:%d", rpcPort, rpcPort),
 		"-v", "/var/run/docker.sock:/var/run/docker.sock",
@@ -431,10 +431,9 @@ func (f *Funnel) killTestServerContainers(ids []string) {
 }
 
 // CleanupTestServerContainer stops the docker container running the test funnel server
-func (f *Funnel) CleanupTestServerContainer() {
+func (f *Funnel) CleanupTestServerContainer(containerName string) {
 	f.Cleanup()
-	s := f.findTestServerContainers()
-	f.killTestServerContainers(s)
+	f.killTestServerContainers([]string{containerName})
 	return
 }
 

diff --git a/tests/gridengine/gridengine_test.go b/tests/gridengine/gridengine_test.go
@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"os"
+	"os/exec"
 	"testing"
 
 	"github.com/ohsu-comp-bio/funnel/logger"
@@ -11,6 +12,7 @@ import (
 )
 
 var fun *tests.Funnel
+var serverName string
 
 func TestMain(m *testing.M) {
 	conf := tests.DefaultConfig()
@@ -21,8 +23,9 @@ func TestMain(m *testing.M) {
 	}
 
 	fun = tests.NewFunnel(conf)
-	fun.StartServerInDocker("ohsucompbio/gridengine:latest", []string{})
-	defer fun.CleanupTestServerContainer()
+	serverName = "funnel-test-server-" + tests.RandomString(6)
+	fun.StartServerInDocker(serverName, "ohsucompbio/gridengine:latest", []string{})
+	defer fun.CleanupTestServerContainer(serverName)
 
 	m.Run()
 	return
@@ -69,8 +72,16 @@ func TestCancel(t *testing.T) {
 	}
 
 	task := fun.Wait(id)
-
 	if task.State != tes.State_CANCELED {
-		t.Fatal("expected task to get canceled")
+		t.Error("expected task to get canceled")
+	}
+
+	bid := task.Logs[0].Metadata["gridengine_id"]
+	cmd := exec.Command("docker", "exec", "-e", "SGE_ROOT=/usr/share/gridengine", serverName, "qstat", "-j", bid)
+	out, err := cmd.Output()
+	t.Log("cmd output:", string(out))
+	t.Log("error:", err)
+	if err == nil {
+		t.Error("expected error from qstat command")
 	}
 }
diff --git a/tests/htcondor/htcondor_test.go b/tests/htcondor/htcondor_test.go
@@ -3,6 +3,8 @@ package htcondor
 import (
 	"context"
 	"os"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"github.com/ohsu-comp-bio/funnel/logger"
@@ -11,6 +13,7 @@ import (
 )
 
 var fun *tests.Funnel
+var serverName string
 
 func TestMain(m *testing.M) {
 	conf := tests.DefaultConfig()
@@ -21,8 +24,9 @@ func TestMain(m *testing.M) {
 	}
 
 	fun = tests.NewFunnel(conf)
-	fun.StartServerInDocker("ohsucompbio/htcondor:latest", []string{})
-	defer fun.CleanupTestServerContainer()
+	serverName = "funnel-test-server-" + tests.RandomString(6)
+	fun.StartServerInDocker(serverName, "ohsucompbio/htcondor:latest", []string{})
+	defer fun.CleanupTestServerContainer(serverName)
 
 	m.Run()
 	return
@@ -69,8 +73,16 @@ func TestCancel(t *testing.T) {
 	}
 
 	task := fun.Wait(id)
-
 	if task.State != tes.State_CANCELED {
-		t.Fatal("expected task to get canceled")
+		t.Error("expected task to get canceled")
+	}
+
+	bid := task.Logs[0].Metadata["htcondor_id"]
+	cmd := exec.Command("docker", "exec", serverName, "condor_q", bid)
+	out, err := cmd.Output()
+	t.Log("cmd output:", string(out))
+	t.Log("error:", err)
+	if !strings.Contains(string(out), "0 jobs;") {
+		t.Error("unexpected condor_q output")
 	}
 }
diff --git a/tests/pbs/pbs_test.go b/tests/pbs/pbs_test.go
@@ -10,6 +10,7 @@ import (
 )
 
 var fun *tests.Funnel
+var serverName string
 
 func TestMain(m *testing.M) {
 	conf := tests.DefaultConfig()
@@ -20,8 +21,9 @@ func TestMain(m *testing.M) {
 	}
 
 	fun = tests.NewFunnel(conf)
-	fun.StartServerInDocker("ohsucompbio/pbs-torque:latest", []string{"--hostname", "docker", "--privileged"})
-	defer fun.CleanupTestServerContainer()
+	serverName = "funnel-test-server-" + tests.RandomString(6)
+	fun.StartServerInDocker(serverName, "ohsucompbio/pbs-torque:latest", []string{"--hostname", "docker", "--privileged"})
+	defer fun.CleanupTestServerContainer(serverName)
 
 	m.Run()
 	return

diff --git a/tests/slurm/slurm_test.go b/tests/slurm/slurm_test.go
@@ -3,6 +3,8 @@ package slurm
 import (
 	"context"
 	"os"
+	"os/exec"
+	"strings"
 	"testing"
 
 	"github.com/ohsu-comp-bio/funnel/logger"
@@ -11,6 +13,7 @@ import (
 )
 
 var fun *tests.Funnel
+var serverName string
 
 func TestMain(m *testing.M) {
 	conf := tests.DefaultConfig()
@@ -21,8 +24,9 @@ func TestMain(m *testing.M) {
 	}
 
 	fun = tests.NewFunnel(conf)
-	fun.StartServerInDocker("ohsucompbio/slurm:latest", []string{"--hostname", "ernie"})
-	defer fun.CleanupTestServerContainer()
+	serverName = "funnel-test-server-" + tests.RandomString(6)
+	fun.StartServerInDocker(serverName, "ohsucompbio/slurm:latest", []string{"--hostname", "ernie"})
+	defer fun.CleanupTestServerContainer(serverName)
 
 	m.Run()
 	return
@@ -80,8 +84,16 @@ func TestCancel(t *testing.T) {
 	}
 
 	task := fun.Wait(id)
-
 	if task.State != tes.State_CANCELED {
-		t.Fatal("expected task to get canceled")
+		t.Error("expected task to get canceled")
+	}
+
+	bid := task.Logs[0].Metadata["slurm_id"]
+	cmd := exec.Command("docker", "exec", serverName, "squeue", "--job", bid)
+	out, err := cmd.Output()
+	t.Log("cmd output:", string(out))
+	t.Log("error:", err)
+	if strings.Contains(string(out), bid) {
+		t.Error("unexpected squeue output")
 	}
 }