sim/pyspec: bubble error

simulators/ethereum/pyspec/main.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strconv"
 	"strings"
 	"sync"
@@ -90,8 +91,11 @@ func fixtureRunner(t *hivesim.T) {
 		}()
 	}
 
+	_, testPattern := t.Sim.TestPattern()
+	re := regexp.MustCompile(testPattern)
+
 	// deliver and run test cases against each client
-	loadFixtureTests(t, fileRoot, func(tc testcase) {
+	loadFixtureTests(t, fileRoot, re, func(tc testcase) {
 		for _, client := range clientTypes {
 			if !client.HasRole("eth1") {
 				continue

simulators/ethereum/pyspec/runner.go

@@ -4,8 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
+	"io/fs"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
@@ -18,16 +19,21 @@ import (
 
 // loadFixtureTests extracts tests from fixture.json files in a given directory,
 // creates a testcase for each test, and passes the testcase struct to fn.
-func loadFixtureTests(t *hivesim.T, root string, fn func(testcase)) {
-	filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+func loadFixtureTests(t *hivesim.T, root string, re *regexp.Regexp, fn func(testcase)) {
+	filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
 		// check file is actually a fixture
 		if err != nil {
 			t.Logf("unable to walk path: %s", err)
 			return err
 		}
-		if info.IsDir() || !strings.HasSuffix(info.Name(), ".json") {
+		if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") {
 			return nil
 		}
+		pathname := strings.TrimSuffix(strings.TrimPrefix(path, root), ".json")
+		if !re.MatchString(pathname) {
+			fmt.Println("skip", pathname)
+			return nil // skip
+		}
 		excludePaths := []string{"example/"} // modify for tests to exclude
 		if strings.Contains(path, strings.Join(excludePaths, "")) {
 			return nil
@@ -54,7 +60,10 @@ func loadFixtureTests(t *hivesim.T, root string, fn func(testcase)) {
 				filepath: path,
 			}
 			// extract genesis, payloads & post allocation field to tc
-			tc.extractFixtureFields(fixture.json)
+			if err := tc.extractFixtureFields(fixture.json); err != nil {
+				t.Logf("test %v / %v: unable to extract fixture fields: %v", d.Name(), name, err)
+				tc.failedErr = fmt.Errorf("unable to extract fixture fields: %v", err)
+			}
 			// feed tc to single worker within fixtureRunner()
 			fn(tc)
 		}
@@ -85,7 +94,11 @@ func (tc *testcase) run(t *hivesim.T) {
 	}
 	tc.updateEnv(env)
 	t0 := time.Now()
-
+	// If test is already failed, don't bother spinning up a client
+	if tc.failedErr != nil {
+		t.Errorf("test failed early: %v", tc.failedErr)
+		return
+	}
 	// start client (also creates an engine RPC client internally)
 	t.Log("starting client with Engine API.")
 	engineClient, err := engineStarter.StartClient(t, ctx, tc.genesis, env, nil)

simulators/ethereum/pyspec/types.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"math/big"
 
 	api "github.com/ethereum/go-ethereum/beacon/engine"
@@ -144,21 +145,25 @@ type withdrawalsUnmarshaling struct {
 
 // extractFixtureFields extracts the genesis, payloads and post allocation
 // fields from the given fixture test and stores them in the testcase struct.
-func (tc *testcase) extractFixtureFields(fixture fixtureJSON) {
+func (tc *testcase) extractFixtureFields(fixture fixtureJSON) error {
 	// extract genesis fields from fixture test
 	tc.genesis = extractGenesis(fixture)
 
 	// extract payloads from each block
 	payloads := []*api.ExecutableData{}
-	for _, block := range fixture.Blocks {
-		block, _ := block.decodeBlock()
+	for _, bl := range fixture.Blocks {
+		block, err := bl.decodeBlock()
+		if err != nil {
+			return fmt.Errorf("failed to decode block: %v", err)
+		}
 		payload := api.BlockToExecutableData(block, common.Big0).ExecutionPayload
 		payloads = append(payloads, payload)
 	}
 	tc.payloads = payloads
 
 	// extract post account information
 	tc.postAlloc = &fixture.Post
+	return nil
 }
 
 // extractGenesis extracts the genesis block information from the given fixture