diff --git a/simulators/ethereum/pyspec/main.go b/simulators/ethereum/pyspec/main.go index 151ed13772..5a3f14585c 100644 --- a/simulators/ethereum/pyspec/main.go +++ b/simulators/ethereum/pyspec/main.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "strconv" "strings" "sync" @@ -90,8 +91,11 @@ func fixtureRunner(t *hivesim.T) { }() } + _, testPattern := t.Sim.TestPattern() + re := regexp.MustCompile(testPattern) + // deliver and run test cases against each client - loadFixtureTests(t, fileRoot, func(tc testcase) { + loadFixtureTests(t, fileRoot, re, func(tc testcase) { for _, client := range clientTypes { if !client.HasRole("eth1") { continue diff --git a/simulators/ethereum/pyspec/runner.go b/simulators/ethereum/pyspec/runner.go index 83bbdb5f27..a8eaa588cb 100644 --- a/simulators/ethereum/pyspec/runner.go +++ b/simulators/ethereum/pyspec/runner.go @@ -4,8 +4,9 @@ import ( "context" "errors" "fmt" - "os" + "io/fs" "path/filepath" + "regexp" "strings" "time" @@ -18,16 +19,21 @@ import ( // loadFixtureTests extracts tests from fixture.json files in a given directory, // creates a testcase for each test, and passes the testcase struct to fn. -func loadFixtureTests(t *hivesim.T, root string, fn func(testcase)) { - filepath.Walk(root, func(path string, info os.FileInfo, err error) error { +func loadFixtureTests(t *hivesim.T, root string, re *regexp.Regexp, fn func(testcase)) { + filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { // check file is actually a fixture if err != nil { t.Logf("unable to walk path: %s", err) return err } - if info.IsDir() || !strings.HasSuffix(info.Name(), ".json") { + if d.IsDir() || !strings.HasSuffix(d.Name(), ".json") { return nil } + pathname := strings.TrimSuffix(strings.TrimPrefix(path, root), ".json") + if !re.MatchString(pathname) { + fmt.Println("skip", pathname) + return nil // skip + } excludePaths := []string{"example/"} // modify for tests to exclude if strings.Contains(path, strings.Join(excludePaths, "")) { return nil @@ -54,7 +60,10 @@ func loadFixtureTests(t *hivesim.T, root string, fn func(testcase)) { filepath: path, } // extract genesis, payloads & post allocation field to tc - tc.extractFixtureFields(fixture.json) + if err := tc.extractFixtureFields(fixture.json); err != nil { + t.Logf("test %v / %v: unable to extract fixture fields: %v", d.Name(), name, err) + tc.failedErr = fmt.Errorf("unable to extract fixture fields: %v", err) + } // feed tc to single worker within fixtureRunner() fn(tc) } @@ -85,7 +94,11 @@ func (tc *testcase) run(t *hivesim.T) { } tc.updateEnv(env) t0 := time.Now() - + // If test is already failed, don't bother spinning up a client + if tc.failedErr != nil { + t.Errorf("test failed early: %v", tc.failedErr) + return + } // start client (also creates an engine RPC client internally) t.Log("starting client with Engine API.") engineClient, err := engineStarter.StartClient(t, ctx, tc.genesis, env, nil) diff --git a/simulators/ethereum/pyspec/types.go b/simulators/ethereum/pyspec/types.go index 960b301b58..83f011ac15 100644 --- a/simulators/ethereum/pyspec/types.go +++ b/simulators/ethereum/pyspec/types.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "fmt" "math/big" api "github.com/ethereum/go-ethereum/beacon/engine" @@ -144,14 +145,17 @@ type withdrawalsUnmarshaling struct { // extractFixtureFields extracts the genesis, payloads and post allocation // fields from the given fixture test and stores them in the testcase struct. -func (tc *testcase) extractFixtureFields(fixture fixtureJSON) { +func (tc *testcase) extractFixtureFields(fixture fixtureJSON) error { // extract genesis fields from fixture test tc.genesis = extractGenesis(fixture) // extract payloads from each block payloads := []*api.ExecutableData{} - for _, block := range fixture.Blocks { - block, _ := block.decodeBlock() + for _, bl := range fixture.Blocks { + block, err := bl.decodeBlock() + if err != nil { + return fmt.Errorf("failed to decode block: %v", err) + } payload := api.BlockToExecutableData(block, common.Big0).ExecutionPayload payloads = append(payloads, payload) } @@ -159,6 +163,7 @@ func (tc *testcase) extractFixtureFields(fixture fixtureJSON) { // extract post account information tc.postAlloc = &fixture.Post + return nil } // extractGenesis extracts the genesis block information from the given fixture