Add support for external test harness (#343)

Signed-off-by: Nadav Samet <[email protected]>
bufbuild · Jun 9, 2020 · 1457b8e · 1457b8e
1 parent 278964a
commit 1457b8e
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 12 deletions.
diff --git a/tests/harness/executor/executor.go b/tests/harness/executor/executor.go
@@ -21,10 +21,12 @@ func main() {
 	ccFlag := flag.Bool("cc", false, "Run c++ test harness")
 	javaFlag := flag.Bool("java", false, "Run java test harness")
 	pythonFlag := flag.Bool("python", false, "Run python test harness")
+	externalHarnessFlag := flag.String("external_harness", "", "Path to a binary to be executed as an external test harness")
 	flag.Parse()
 
 	start := time.Now()
-	successes, failures, skips := run(*parallelism, *goFlag, *gogoFlag, *ccFlag, *javaFlag, *pythonFlag)
+	harnesses := Harnesses(*goFlag, *gogoFlag, *ccFlag, *javaFlag, *pythonFlag, *externalHarnessFlag)
+	successes, failures, skips := run(*parallelism, harnesses)
 
 	log.Printf("Successes: %d | Failures: %d | Skips: %d (%v)",
 		successes, failures, skips, time.Since(start))
@@ -34,12 +36,12 @@ func main() {
 	}
 }
 
-func run(parallelism int, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) (successes, failures, skips uint64) {
+func run(parallelism int, harnesses []Harness) (successes, failures, skips uint64) {
 	wg := new(sync.WaitGroup)
 	if parallelism <= 0 {
 		panic("Parallelism must be > 0")
 	}
-	if !(goFlag || gogoFlag || ccFlag || javaFlag || pythonFlag) {
+	if len(harnesses) == 0 {
 		panic("At least one harness must be selected with a flag")
 	}
 	wg.Add(parallelism)
@@ -49,7 +51,7 @@ func run(parallelism int, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool
 	done := make(chan struct{})
 
 	for i := 0; i < parallelism; i++ {
-		go Work(wg, in, out, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
+		go Work(wg, in, out, harnesses)
 	}
 
 	go func() {

diff --git a/tests/harness/executor/harness.go b/tests/harness/executor/harness.go
@@ -12,12 +12,12 @@ import (
 
 	"strings"
 
-	"github.com/golang/protobuf/proto"
 	harness "github.com/envoyproxy/protoc-gen-validate/tests/harness/go"
+	"github.com/golang/protobuf/proto"
 	"golang.org/x/net/context"
 )
 
-func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) []Harness {
+func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool, externalHarnessFlag string) []Harness {
 	harnesses := make([]Harness, 0)
 	if goFlag {
 		harnesses = append(harnesses, InitHarness("tests/harness/go/main/go-harness"))
@@ -34,6 +34,9 @@ func Harnesses(goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFla
 	if pythonFlag {
 		harnesses = append(harnesses, InitHarness("tests/harness/python/python-harness"))
 	}
+	if externalHarnessFlag != "" {
+		harnesses = append(harnesses, InitHarness(externalHarnessFlag))
+	}
 	return harnesses
 }
 

diff --git a/tests/harness/executor/worker.go b/tests/harness/executor/worker.go
@@ -8,20 +8,20 @@ import (
 	"sync"
 	"time"
 
+	harness "github.com/envoyproxy/protoc-gen-validate/tests/harness/go"
 	"github.com/golang/protobuf/proto"
 	"github.com/golang/protobuf/ptypes"
-	harness "github.com/envoyproxy/protoc-gen-validate/tests/harness/go"
 )
 
-func Work(wg *sync.WaitGroup, in <-chan TestCase, out chan<- TestResult, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) {
+func Work(wg *sync.WaitGroup, in <-chan TestCase, out chan<- TestResult, harnesses []Harness) {
 	for tc := range in {
-		ok, skip := execTestCase(tc, goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
+		ok, skip := execTestCase(tc, harnesses)
 		out <- TestResult{ok, skip}
 	}
 	wg.Done()
 }
 
-func execTestCase(tc TestCase, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag bool, pythonFlag bool) (ok, skip bool) {
+func execTestCase(tc TestCase, harnesses []Harness) (ok, skip bool) {
 	any, err := ptypes.MarshalAny(tc.Message)
 	if err != nil {
 		log.Printf("unable to convert test case %q to Any - %v", tc.Name, err)
@@ -37,8 +37,6 @@ func execTestCase(tc TestCase, goFlag bool, gogoFlag bool, ccFlag bool, javaFlag
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()
 
-	harnesses := Harnesses(goFlag, gogoFlag, ccFlag, javaFlag, pythonFlag)
-
 	wg := new(sync.WaitGroup)
 	wg.Add(len(harnesses))