Add support for external test harness

Signed-off-by: Nadav Samet <[email protected]>
bufbuild · Jun 8, 2020 · 8a77620 · 8a77620
1 parent 478e95e
commit 8a77620
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 10 deletions.
diff --git a/tests/harness/executor/executor.go b/tests/harness/executor/executor.go
@@ -21,10 +21,11 @@ func main() {
 	ccFlag := flag.Bool("cc", false, "Run c++ test harness")
 	javaFlag := flag.Bool("java", false, "Run java test harness")
 	pythonFlag := flag.Bool("python", false, "Run python test harness")
+	externalFlag := flag.String("external", "", "Run an external test harness through the given executable")
 	flag.Parse()
 
 	start := time.Now()
-	successes, failures, skips := run(*parallelism, *goFlag, *gogoFlag, *ccFlag, *javaFlag, *pythonFlag)
+	successes, failures, skips := run(*parallelism, *goFlag, *gogoFlag, *ccFlag, *javaFlag, *pythonFlag, *externalFlag)
 
 	log.Printf("Successes: %d | Failures: %d | Skips: %d (%v)",
 		successes, failures, skips, time.Since(start))
@@ -34,12 +35,12 @@ func main() {
 	}
 }
 
-func run(parallelism int, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) (successes, failures, skips uint64) {
+func run(parallelism int, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool, externalFlag string) (successes, failures, skips uint64) {
 	wg := new(sync.WaitGroup)
 	if parallelism <= 0 {
 		panic("Parallelism must be > 0")
 	}
-	if !(goFlag || gogoFlag || ccFlag || javaFlag || pythonFlag) {
+	if !(goFlag || gogoFlag || ccFlag || javaFlag || pythonFlag || externalFlag != "") {
 		panic("At least one harness must be selected with a flag")
 	}
 	wg.Add(parallelism)
@@ -49,7 +50,7 @@ func run(parallelism int, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool
 	done := make(chan struct{})
 
 	for i := 0; i < parallelism; i++ {
-		go Work(wg, in, out, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
+		go Work(wg, in, out, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag, externalFlag)
 	}
 
 	go func() {

diff --git a/tests/harness/executor/harness.go b/tests/harness/executor/harness.go
@@ -17,7 +17,7 @@ import (
 	"golang.org/x/net/context"
 )
 
-func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) []Harness {
+func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool, externalFlag string) []Harness {
 	harnesses := make([]Harness, 0)
 	if goFlag {
 		harnesses = append(harnesses, InitHarness("tests/harness/go/main/go-harness"))
@@ -34,6 +34,9 @@ func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFla
 	if pythonFlag {
 		harnesses = append(harnesses, InitHarness("tests/harness/python/python-harness"))
 	}
+	if externalFlag != "" {
+		harnesses = append(harnesses, InitHarness(externalFlag))
+	}
 	return harnesses
 }
 

diff --git a/tests/harness/executor/worker.go b/tests/harness/executor/worker.go
@@ -8,20 +8,20 @@ import (
 	"sync"
 	"time"
 
+	harness "github.com/envoyproxy/protoc-gen-validate/tests/harness/go"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
-	harness "github.com/envoyproxy/protoc-gen-validate/tests/harness/go"
 )
 
-func Work(wg *sync.WaitGroup, in <-chan TestCase, out chan<- TestResult, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) {
+func Work(wg *sync.WaitGroup, in <-chan TestCase, out chan<- TestResult, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool, externalFlag string) {
 	for tc := range in {
-		ok, skip := execTestCase(tc, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
+		ok, skip := execTestCase(tc, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag, externalFlag)
 		out <- TestResult{ok, skip}
 	}
 	wg.Done()
 }
 
-func execTestCase(tc TestCase, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) (ok, skip bool) {
+func execTestCase(tc TestCase, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool, externalFlag string) (ok, skip bool) {
 	any, err := ptypes.MarshalAny(tc.Message)
 	if err != nil {
 		log.Printf("unable to convert test case %q to Any - %v", tc.Name, err)
@@ -37,7 +37,7 @@ func execTestCase(tc TestCase, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 
-	harnesses := Harnesses(goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
+	harnesses := Harnesses(goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag, externalFlag)
 
 	wg := new(sync.WaitGroup)
 	wg.Add(len(harnesses))