feat(verification): Expose verification output

BREAKING CHANGE: PactClient#VerifyProvider now returns a
types.ProviderVerifierResponse as well as an error. An error
means that the client was unable to run the verification code
successfully. The ProviderVerifierResponse is a direct mapping
of the json output returned by the pact-provider-verifier binary.
Iterate over the Examples structs to determine if they passed.

NOTE: Verbose output for the types.VerifyRequest is not compatible
with the json format since it logs to stdout, causing the json to
be unparsable.

daemon/daemon.go

@@ -15,7 +15,9 @@ package daemon
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net"
 	"net/http"
@@ -108,34 +110,61 @@ func (d Daemon) StartServer(request types.MockServer, reply *types.MockServer) e
 }
 
 // VerifyProvider runs the Pact Provider Verification Process.
-func (d Daemon) VerifyProvider(request types.VerifyRequest, reply *types.CommandResponse) error {
+func (d Daemon) VerifyProvider(request types.VerifyRequest, reply *types.ProviderVerifierResponse) error {
 	log.Println("[DEBUG] daemon - verifying provider")
-	exitCode := 1
 
 	// Convert request into flags, and validate request
 	err := request.Validate()
 	if err != nil {
-		*reply = types.CommandResponse{
-			ExitCode: exitCode,
-			Message:  err.Error(),
-		}
-		return nil
+		return err
 	}
 
-	var out bytes.Buffer
+	// Run command, splitting out stderr and stdout. The command can fail for
+	// several reasons:
+	// 1. Command is unable to run at all.
+	// 2. Command runs, but fails for unknown reason.
+	// 3. Command runs, and returns exit status 1 because the tests fail.
+	//
+	// First, attempt to decode the response of the stdout.
+	// If that is successful, we are at case 3. Return stdout as message, no error.
+	// Else, return an error, include stderr and stdout in both the error and message.
 	svc := d.verificationSvcManager.NewService(request.Args)
-	cmd, err := svc.Run(&out)
+	cmd := svc.Command()
+
+	stdOutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	stdErrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+	stdOut, err := ioutil.ReadAll(stdOutPipe)
+	if err != nil {
+		return err
+	}
+	stdErr, err := ioutil.ReadAll(stdErrPipe)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
 
-	if err == nil && cmd.ProcessState != nil && cmd.ProcessState.Success() {
-		exitCode = 0
+	decoder := json.NewDecoder(bytes.NewReader(stdOut))
+	dErr := decoder.Decode(&reply)
+	if dErr == nil {
+		return nil
 	}
 
-	*reply = types.CommandResponse{
-		ExitCode: exitCode,
-		Message:  string(out.Bytes()),
+	if err == nil {
+		err = dErr
 	}
 
-	return nil
+	return fmt.Errorf("error verifying provider: %s\n\nSTDERR:\n%s\n\nSTDOUT:\n%s", err, stdErr, stdOut)
 }
 
 // ListServers returns a slice of all running types.MockServers.

daemon/daemon_test.go

@@ -27,7 +27,7 @@ func createMockedDaemon(success bool) (*Daemon, *ServiceMock) {
 		execFunc = fakeExecFailCommand
 	}
 	svc := &ServiceMock{
-		Command:           "test",
+		Cmd:               "test",
 		Args:              []string{},
 		ServiceStopResult: true,
 		ServiceStopError:  nil,
@@ -245,19 +245,18 @@ func TestStopServer_FailedStatus(t *testing.T) {
 func TestVerifyProvider_MissingProviderBaseURL(t *testing.T) {
 	daemon, _ := createMockedDaemon(true)
 
-	req := types.VerifyRequest{}
-	res := types.CommandResponse{}
+	req := types.VerifyRequest{
+		PactURLs: []string{"url1", "url2"},
+	}
+	res := types.ProviderVerifierResponse{}
 	err := daemon.VerifyProvider(req, &res)
 
-	if err != nil {
-		t.Fatalf("Error: %v", err)
-	}
-	if res.ExitCode != 1 {
-		t.Fatalf("Expected non-zero exit code (1) but got: %d", res.ExitCode)
+	if err == nil {
+		t.Fatal("Expected an error")
 	}
 
-	if !strings.Contains(res.Message, "ProviderBaseURL is mandatory") {
-		t.Fatalf("Expected error message but got '%s'", res.Message)
+	if !strings.Contains(err.Error(), "ProviderBaseURL is mandatory") {
+		t.Fatalf("Expected error message but got '%s'", err.Error())
 	}
 }
 
@@ -267,18 +266,15 @@ func TestVerifyProvider_MissingPactURLs(t *testing.T) {
 	req := types.VerifyRequest{
 		ProviderBaseURL: "http://foo.com",
 	}
-	res := types.CommandResponse{}
+	res := types.ProviderVerifierResponse{}
 	err := daemon.VerifyProvider(req, &res)
 
-	if err != nil {
-		t.Fatalf("Error: %v", err)
-	}
-	if res.ExitCode != 1 {
-		t.Fatalf("Expected non-zero exit code (1) but got: %d", res.ExitCode)
+	if err == nil {
+		t.Fatal("Expected an error")
 	}
 
-	if !strings.Contains(res.Message, "PactURLs is mandatory") {
-		t.Fatalf("Expected error message but got '%s'", res.Message)
+	if !strings.Contains(err.Error(), "PactURLs is mandatory") {
+		t.Fatalf("Expected error message but got '%s'", err.Error())
 	}
 }
 
@@ -289,10 +285,10 @@ func TestVerifyProvider_Valid(t *testing.T) {
 		ProviderBaseURL: "http://foo.com",
 		PactURLs:        []string{"foo.json", "bar.json"},
 	}
-	res := types.CommandResponse{}
+	res := types.ProviderVerifierResponse{}
 	err := daemon.VerifyProvider(req, &res)
 	if err != nil {
-		t.Fatalf("Error: %v", err)
+		t.Fatalf("Error: %s", err)
 	}
 }
 
@@ -303,17 +299,14 @@ func TestVerifyProvider_FailedCommand(t *testing.T) {
 		ProviderBaseURL: "http://foo.com",
 		PactURLs:        []string{"foo.json", "bar.json"},
 	}
-	res := types.CommandResponse{}
+	res := types.ProviderVerifierResponse{}
 	err := daemon.VerifyProvider(req, &res)
-	if err != nil {
-		t.Fatalf("Error: %v", err)
-	}
-	if res.ExitCode != 1 {
-		t.Fatalf("Expected non-zero exit code (1) but got: %d", res.ExitCode)
+	if err == nil {
+		t.Fatal("Expected an error")
 	}
 
-	if !strings.Contains(res.Message, "COMMAND: oh noes!") {
-		t.Fatalf("Expected error message but got '%s'", res.Message)
+	if !strings.Contains(err.Error(), "COMMAND: oh noes!") {
+		t.Fatalf("Expected error message but got '%s'", err.Error())
 	}
 }
 
@@ -328,7 +321,7 @@ func TestVerifyProvider_ValidProviderStates(t *testing.T) {
 		ProviderStatesURL:      "http://foo/states",
 		ProviderStatesSetupURL: "http://foo/states/setup",
 	}
-	res := types.CommandResponse{}
+	res := types.ProviderVerifierResponse{}
 	err := daemon.VerifyProvider(req, &res)
 	if err != nil {
 		t.Fatalf("Error: %v", err)
@@ -437,19 +430,15 @@ func TestRPCClient_Verify(t *testing.T) {
 		ProviderBaseURL: "http://foo.com",
 		PactURLs:        []string{"foo.json", "bar.json"},
 	}
-	res := types.CommandResponse{}
+	res := types.ProviderVerifierResponse{}
 
 	err = client.Call("Daemon.VerifyProvider", req, &res)
 	if err != nil {
 		log.Fatal("rpc error:", err)
 	}
 
-	if res.ExitCode != 0 {
-		t.Fatalf("Expected exit code of zero but got: %d", res.ExitCode)
-	}
-
-	if res.Message != "COMMAND: oh yays!" {
-		t.Fatalf("Expected empty message but got: '%s'", res.Message)
+	if got, want := res.SummaryLine, "1 examples, 0 failures"; got != want {
+		t.Fatalf("Expected a success message but got: '%s'", got)
 	}
 }
 
@@ -484,6 +473,6 @@ func TestHelperProcess(t *testing.T) {
 	}
 
 	// Success :)
-	fmt.Fprintf(os.Stdout, "COMMAND: oh yays!")
+	fmt.Fprintf(os.Stdout, `{"summary_line":"1 examples, 0 failures"}`)
 	os.Exit(0)
 }

daemon/mock_service.go

@@ -19,7 +19,7 @@ func (m *MockService) NewService(args []string) Service {
 	}
 	m.Args = append(m.Args, args...)
 
-	m.Command = getMockServiceCommandPath()
+	m.Cmd = getMockServiceCommandPath()
 	return m
 }
 

daemon/service.go

@@ -11,6 +11,7 @@ type Service interface {
 	Setup()
 	Stop(pid int) (bool, error)
 	List() map[int]*exec.Cmd
+	Command() *exec.Cmd
 	Start() *exec.Cmd
 	Run(io.Writer) (*exec.Cmd, error)
 	NewService(args []string) Service

daemon/service_manager.go

@@ -6,12 +6,13 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 )
 
 // ServiceManager is the default implementation of the Service interface.
 type ServiceManager struct {
-	Command             string
+	Cmd                 string
 	processes           map[int]*exec.Cmd
 	Args                []string
 	Env                 []string
@@ -102,7 +103,8 @@ func (s *ServiceManager) List() map[int]*exec.Cmd {
 // Run runs a service synchronously and log its output to the given Pipe.
 func (s *ServiceManager) Run(w io.Writer) (*exec.Cmd, error) {
 	log.Println("[DEBUG] starting service")
-	cmd := exec.Command(s.Command, s.Args...)
+	log.Printf("[DEBUG] %s %s\n", s.Cmd, strings.Join(s.Args, " "))
+	cmd := exec.Command(s.Cmd, s.Args...)
 	cmd.Env = s.Env
 	cmd.Stdout = w
 	cmd.Stderr = w
@@ -111,10 +113,16 @@ func (s *ServiceManager) Run(w io.Writer) (*exec.Cmd, error) {
 	return cmd, err
 }
 
+func (s *ServiceManager) Command() *exec.Cmd {
+	cmd := exec.Command(s.Cmd, s.Args...)
+	cmd.Env = s.Env
+	return cmd
+}
+
 // Start a Service and log its output.
 func (s *ServiceManager) Start() *exec.Cmd {
 	log.Println("[DEBUG] starting service")
-	cmd := exec.Command(s.Command, s.Args...)
+	cmd := exec.Command(s.Cmd, s.Args...)
 	cmd.Env = s.Env
 
 	cmdReader, err := cmd.StdoutPipe()

daemon/service_mock.go

@@ -8,7 +8,7 @@ import (
 
 // ServiceMock is the mock implementation of the Service interface.
 type ServiceMock struct {
-	Command             string
+	Cmd                 string
 	processes           map[int]*exec.Cmd
 	Args                []string
 	ServiceStopResult   bool
@@ -65,6 +65,10 @@ func (s *ServiceMock) Start() *exec.Cmd {
 	return cmd
 }
 
+func (s *ServiceMock) Command() *exec.Cmd {
+	return s.ExecFunc()
+}
+
 // NewService creates a new MockService with default settings.
 func (s *ServiceMock) NewService(args []string) Service {
 	return s

daemon/service_test.go

@@ -15,9 +15,9 @@ func createServiceManager() *ServiceManager {
 	cs := []string{"-test.run=TestHelperProcess", "--", os.Args[0]}
 	env := []string{"GO_WANT_HELPER_PROCESS=1", fmt.Sprintf("GO_WANT_HELPER_PROCESS_TO_SUCCEED=true")}
 	mgr := &ServiceManager{
-		Command: os.Args[0],
-		Args:    cs,
-		Env:     env,
+		Cmd:  os.Args[0],
+		Args: cs,
+		Env:  env,
 	}
 	mgr.Setup()
 	return mgr

daemon/verification_service.go

@@ -28,7 +28,7 @@ func (m *VerificationService) NewService(args []string) Service {
 	log.Printf("[DEBUG] starting verification service with args: %v\n", args)
 
 	m.Args = args
-	m.Command = getVerifierCommandPath()
+	m.Cmd = getVerifierCommandPath()
 	return m
 }
 

doc.go

@@ -130,7 +130,7 @@ A typical Provider side test would like something like:
 		}
 		go startMyAPI("http://localhost:8000")
 
-		err := pact.VerifyProvider(types.VerifyRequest{
+		response, err := pact.VerifyProvider(types.VerifyRequest{
 			ProviderBaseURL:        "http://localhost:8000",
 			PactURLs:               []string{"./pacts/my_consumer-my_provider.json"},
 			ProviderStatesSetupURL: "http://localhost:8000/setup",
@@ -139,6 +139,12 @@ A typical Provider side test would like something like:
 		if err != nil {
 			t.Fatal("Error:", err)
 		}
+
+		for _, example := range response.Examples {
+			if example.Status != "passed" {
+				t.Errorf("%s\n%s\n", example.FullDescription, example.Exception.Message)
+			}
+		}
 	}
 
 Note that `PactURLs` can be a list of local pact files or remote based
@@ -155,7 +161,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 
 1. Use "PactURLs" to specify the exact set of pacts to be replayed:
 
-	response = pact.VerifyProvider(types.VerifyRequest{
+	response, err = pact.VerifyProvider(types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		PactURLs:               []string{"http://broker/pacts/provider/them/consumer/me/latest/dev"},
 		ProviderStatesSetupURL: "http://myproviderhost/setup",
@@ -165,7 +171,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 
 2. Use "PactBroker" to automatically find all of the latest consumers:
 
-	response = pact.VerifyProvider(types.VerifyRequest{
+	response, err = pact.VerifyProvider(types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		BrokerURL:              brokerHost,
 		ProviderStatesSetupURL: "http://myproviderhost/setup",
@@ -175,7 +181,7 @@ When validating a Provider, you have 3 options to provide the Pact files:
 
 3. Use "PactBroker" and "Tags" to automatically find all of the latest consumers:
 
-	response = pact.VerifyProvider(types.VerifyRequest{
+	response, err = pact.VerifyProvider(types.VerifyRequest{
 		ProviderBaseURL:        "http://myproviderhost",
 		BrokerURL:              brokerHost,
 		Tags:                   []string{"latest", "sit4"},