cannon: Migrate fuzz tests

cannon/Makefile

@@ -91,9 +91,9 @@ cannon-stf-verify:
 
 fuzz:
 	printf "%s\n" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateConsistencyMulOp ./mipsevm/tests" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateConsistencyMultOp ./mipsevm/tests" \
-		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateConsistencyMultuOp ./mipsevm/tests" \
+		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzMulOp ./mipsevm/tests" \
+		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzMultOp ./mipsevm/tests" \
+		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzMultuOp ./mipsevm/tests" \
 		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateSyscallBrk ./mipsevm/tests" \
 		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateSyscallMmap ./mipsevm/tests" \
 		"go test $(FUZZLDFLAGS) -run NOTAREALTEST -v -fuzztime $(CANNON64_FUZZTIME) -fuzz=FuzzStateSyscallExitGroup ./mipsevm/tests" \

cannon/mipsevm/tests/difftester.go

@@ -20,9 +20,13 @@ import (
 
 type TestNamer[T any] func(testCase T) string
 
+func NoopTestNamer[T any](c T) string {
+	return ""
+}
+
 type SimpleInitializeStateFn func(t require.TestingT, state *multithreaded.State, vm VersionedVMTestCase, r *testutil.RandHelper)
 type SimpleSetExpectationsFn func(t require.TestingT, expect *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult
-type SimplePostStepCheckFn func(t require.TestingT, vm VersionedVMTestCase, deps *TestDependencies)
+type SimplePostStepCheckFn func(t require.TestingT, vm VersionedVMTestCase, deps *TestDependencies, witness *mipsevm.StepWitness)
 
 type soloTestCase struct {
 	name string
@@ -60,8 +64,8 @@ func (d *SimpleDiffTester) SetExpectations(setExpectationsFn SimpleSetExpectatio
 }
 
 func (d *SimpleDiffTester) PostCheck(postStepCheckFn SimplePostStepCheckFn) *SimpleDiffTester {
-	wrappedFn := func(t require.TestingT, testCase soloTestCase, vm VersionedVMTestCase, deps *TestDependencies) {
-		postStepCheckFn(t, vm, deps)
+	wrappedFn := func(t require.TestingT, testCase soloTestCase, vm VersionedVMTestCase, deps *TestDependencies, wit *mipsevm.StepWitness) {
+		postStepCheckFn(t, vm, deps, wit)
 	}
 	d.diffTester.PostCheck(wrappedFn)
 
@@ -77,7 +81,7 @@ func (d *SimpleDiffTester) Run(t *testing.T, opts ...TestOption) {
 
 type InitializeStateFn[T any] func(t require.TestingT, testCase T, state *multithreaded.State, vm VersionedVMTestCase, r *testutil.RandHelper)
 type SetExpectationsFn[T any] func(t require.TestingT, testCase T, expect *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult
-type PostStepCheckFn[T any] func(t require.TestingT, testCase T, vm VersionedVMTestCase, deps *TestDependencies)
+type PostStepCheckFn[T any] func(t require.TestingT, testCase T, vm VersionedVMTestCase, deps *TestDependencies, witness *mipsevm.StepWitness)
 
 type DiffTester[T any] struct {
 	testNamer       TestNamer[T]
@@ -125,43 +129,72 @@ func (d *DiffTester[T]) run(t testRunner, testCases []T, opts ...TestOption) {
 	cfg := newTestConfig(t, opts...)
 	for _, vm := range cfg.vms {
 		for i, testCase := range testCases {
-			randSeed := randomSeed(t, d.testNamer(testCase), i)
+			randSeed := cfg.randomSeed
+			if randSeed == 0 {
+				randSeed = randomSeed(t, d.testNamer(testCase), i)
+			}
 			mods := d.generateTestModifiers(t, testCase, vm, cfg, randSeed)
 			for _, mod := range mods {
 				testName := fmt.Sprintf("%v%v (%v)", d.testNamer(testCase), mod.name, vm.Name)
 				t.Run(testName, func(t testcaseT) {
 					t.Parallel()
 
-					testDeps := cfg.testDependencies()
-					stateOpts := []mtutil.StateOption{mtutil.WithRandomization(randSeed)}
-					stateOpts = append(stateOpts, d.stateOpts...)
-					goVm := vm.VMFactory(testDeps.po, testDeps.stdOut, testDeps.stdErr, testDeps.logger, stateOpts...)
-					state := mtutil.GetMtState(t, goVm)
-
-					// Set up state
-					r := testutil.NewRandHelper(randSeed * 2)
-					d.initState(t, testCase, state, vm, r)
-					mod.stateMod(state)
+					setup := mod.cachedSetup
+					if setup == nil {
+						setup = d.newTestSetup(t, testCase, vm, cfg, randSeed, mod)
+					}
 
+					expect := setup.expect
+					execExpectation := setup.expectedResult
+					var witness *mipsevm.StepWitness
 					for i := 0; i < cfg.steps; i++ {
-						// Set up expectations
-						expect := d.expectedState(t, state)
-						execExpectation := d.setExpectations(t, testCase, expect, vm)
-						mod.expectMod(expect)
+						if i > 0 {
+							// After the initial step, we need to set up our expectations again
+							expect = d.expectedState(t, setup.state)
+							execExpectation = d.setExpectations(t, testCase, expect, vm)
+						}
 
-						execExpectation.assertExpectedResult(t, goVm, vm, expect, cfg)
+						witness = execExpectation.assertExpectedResult(t, setup.goVm, vm, expect, cfg)
 					}
 
 					// Run post-step checks
 					if d.postStepCheck != nil {
-						d.postStepCheck(t, testCase, vm, testDeps)
+						d.postStepCheck(t, testCase, vm, setup.deps, witness)
 					}
 				})
 			}
 		}
 	}
 }
 
+func (d *DiffTester[T]) newTestSetup(t require.TestingT, testCase T, vm VersionedVMTestCase, cfg *TestConfig, randSeed int64, mod *testModifier) *testSetup {
+	testDeps := cfg.testDependencies()
+
+	stateOpts := []mtutil.StateOption{mtutil.WithRandomization(randSeed)}
+	stateOpts = append(stateOpts, d.stateOpts...)
+	goVm := vm.VMFactory(testDeps.po, testDeps.stdOut, testDeps.stdErr, testDeps.logger, stateOpts...)
+
+	state := mtutil.GetMtState(t, goVm)
+	d.initState(t, testCase, state, vm, testutil.NewRandHelper(randSeed*2))
+	if mod != nil {
+		mod.stateMod(state)
+	}
+
+	expect := d.expectedState(t, state)
+	if mod != nil {
+		mod.expectMod(expect)
+	}
+	expectedResult := d.setExpectations(t, testCase, expect, vm)
+
+	return &testSetup{
+		deps:           testDeps,
+		goVm:           goVm,
+		state:          state,
+		expect:         expect,
+		expectedResult: expectedResult,
+	}
+}
+
 func (d *DiffTester[T]) expectedState(t require.TestingT, state *multithreaded.State) *mtutil.ExpectedState {
 	if mtutil.ActiveThreadCount(state) == 0 {
 		// State is invalid, just return an empty expectation
@@ -185,35 +218,32 @@ func (d *DiffTester[T]) isConfigValid(t testRunner) bool {
 }
 
 type testModifier struct {
-	name      string
-	stateMod  func(state *multithreaded.State)
-	expectMod func(expect *mtutil.ExpectedState)
+	name        string
+	stateMod    func(state *multithreaded.State)
+	expectMod   func(expect *mtutil.ExpectedState)
+	cachedSetup *testSetup
 }
 
-func newTestModifier(name string) *testModifier {
+func newTestModifier(name string, cachedSetup *testSetup) *testModifier {
 	return &testModifier{
-		name:      name,
-		stateMod:  func(state *multithreaded.State) {},
-		expectMod: func(expect *mtutil.ExpectedState) {},
+		name:        name,
+		stateMod:    func(state *multithreaded.State) {},
+		expectMod:   func(expect *mtutil.ExpectedState) {},
+		cachedSetup: cachedSetup,
 	}
 }
 
 func (d *DiffTester[T]) generateTestModifiers(t require.TestingT, testCase T, vm VersionedVMTestCase, cfg *TestConfig, randSeed int64) []*testModifier {
+	// Set up state
+	setup := d.newTestSetup(t, testCase, vm, cfg, randSeed, nil)
+
+	// Build modifiers array, start with the original case (noop modification)
 	modifiers := []*testModifier{
-		newTestModifier(""), // Always return a noop
+		newTestModifier("", setup), // Always return a noop
 	}
 
-	// Set up state
-	testDeps := cfg.testDependencies()
-	goVm := vm.VMFactory(testDeps.po, testDeps.stdOut, testDeps.stdErr, testDeps.logger, d.stateOpts...)
-	state := mtutil.GetMtState(t, goVm)
-	d.initState(t, testCase, state, vm, testutil.NewRandHelper(randSeed))
-	// Process expectations
-	expect := d.expectedState(t, state)
-	d.setExpectations(t, testCase, expect, vm)
-
 	// Generate test modifiers based on expectations
-	modifiers = append(modifiers, d.memReservationTestModifier(cfg, randSeed, expect)...)
+	modifiers = append(modifiers, d.memReservationTestModifier(cfg, randSeed, setup.expect)...)
 
 	return modifiers
 }
@@ -294,6 +324,14 @@ func randomSeed(t require.TestingT, s string, extraData ...int) int64 {
 	return int64(h.Sum64())
 }
 
+type testSetup struct {
+	deps           *TestDependencies
+	goVm           mipsevm.FPVM
+	state          *multithreaded.State
+	expect         *mtutil.ExpectedState
+	expectedResult ExpectedExecResult
+}
+
 type TestDependencies struct {
 	po     mipsevm.PreimageOracle
 	stdOut io.Writer
@@ -313,6 +351,8 @@ type TestConfig struct {
 	tracingHooks *tracing.Hooks
 	// Allow consumer to control automated test generation
 	skipAutomaticMemoryReservationTests bool
+	// Allow consumer to configure a random seed, if not configured (equal to 0) one will be generated
+	randomSeed int64
 }
 
 func (c *TestConfig) testDependencies() *TestDependencies {
@@ -344,6 +384,18 @@ func WithVm(vm VersionedVMTestCase) TestOption {
 	}
 }
 
+func WithVms(vms []VersionedVMTestCase) TestOption {
+	return func(tc *TestConfig) {
+		tc.vms = vms
+	}
+}
+
+func WithRandomSeed(seed int64) TestOption {
+	return func(tc *TestConfig) {
+		tc.randomSeed = seed
+	}
+}
+
 // WithTracingHooks Sets tracing hooks - see: testutil.MarkdownTracer
 func WithTracingHooks(hooks *tracing.Hooks) TestOption {
 	return func(tc *TestConfig) {
@@ -362,7 +414,6 @@ func WithSteps(steps int) TestOption {
 
 func newTestConfig(t require.TestingT, opts ...TestOption) *TestConfig {
 	testConfig := &TestConfig{
-		vms:    GetMipsVersionTestCases(t),
 		po:     func() mipsevm.PreimageOracle { return nil },
 		stdOut: func() io.Writer { return os.Stdout },
 		stdErr: func() io.Writer { return os.Stderr },
@@ -373,11 +424,17 @@ func newTestConfig(t require.TestingT, opts ...TestOption) *TestConfig {
 	for _, opt := range opts {
 		opt(testConfig)
 	}
+
+	// Generating vm versions is expensive, only do it if necessary
+	if testConfig.vms == nil {
+		testConfig.vms = GetMipsVersionTestCases(t)
+	}
+
 	return testConfig
 }
 
 type ExpectedExecResult interface {
-	assertExpectedResult(t testing.TB, vm mipsevm.FPVM, vmType VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig)
+	assertExpectedResult(t testing.TB, vm mipsevm.FPVM, vmType VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) *mipsevm.StepWitness
 }
 
 type normalExecResult struct{}
@@ -386,7 +443,7 @@ func ExpectNormalExecution() ExpectedExecResult {
 	return normalExecResult{}
 }
 
-func (e normalExecResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) {
+func (e normalExecResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) *mipsevm.StepWitness {
 	// Step the VM
 	state := goVm.GetState()
 	step := state.GetStep()
@@ -396,6 +453,8 @@ func (e normalExecResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM,
 	// Validate
 	expect.Validate(t, state)
 	testutil.ValidateEVM(t, stepWitness, step, goVm, vmVersion.StateHashFn, vmVersion.Contracts)
+
+	return stepWitness
 }
 
 type vmPanicResult struct {
@@ -441,7 +500,7 @@ func ExpectVmPanicWithCustomErr(goPanicMsg interface{}, customErrSignature strin
 	return result
 }
 
-func (e vmPanicResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) {
+func (e vmPanicResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) *mipsevm.StepWitness {
 	state := goVm.GetState()
 	proofData := e.proofData
 	if proofData == nil {
@@ -456,6 +515,8 @@ func (e vmPanicResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmV
 	} else {
 		t.Fatalf("Invalid panic value provided.  Go panic value must be a string or error.  Got: %v", e.panicValue)
 	}
+
+	return nil
 }
 
 type preimageOracleRevertResult struct {
@@ -474,9 +535,10 @@ func ExpectPreimageOraclePanic(preimageKey [32]byte, preimageValue []byte, preim
 	}
 }
 
-func (e preimageOracleRevertResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) {
+func (e preimageOracleRevertResult) assertExpectedResult(t testing.TB, goVm mipsevm.FPVM, vmVersion VersionedVMTestCase, expect *mtutil.ExpectedState, cfg *TestConfig) *mipsevm.StepWitness {
 	require.PanicsWithValue(t, e.panicMsg, func() { _, _ = goVm.Step(true) })
 	testutil.AssertPreimageOracleReverts(t, e.preimageKey, e.preimageValue, e.preimageOffset, vmVersion.Contracts)
+	return nil
 }
 
 type testcaseT interface {

cannon/mipsevm/tests/difftester_test.go

@@ -53,8 +53,7 @@ func TestDiffTester_Run_SimpleTest(t *testing.T) {
 
 			// Validate that we invoked initState and setExpectations as expected
 			for _, c := range testCases {
-				// Difftester runs extra calls in order to analyze the tests
-				expectedCalls := 2 * len(versions)
+				expectedCalls := len(versions)
 				require.Equal(t, expectedCalls, initStateCalled[c.name])
 				require.Equal(t, expectedCalls, expectationsCalled[c.name])
 			}
@@ -115,9 +114,8 @@ func TestDiffTester_Run_WithSteps(t *testing.T) {
 
 			// Validate that we invoked initState and setExpectations as expected
 			for _, c := range cases {
-				// Difftester runs extra calls in order to analyze the tests
-				initCalls := 2 * len(versions)
-				expectCalls := oc.expectedSteps*len(versions) + len(versions)
+				initCalls := len(versions)
+				expectCalls := oc.expectedSteps * len(versions)
 				require.Equal(t, initCalls, initStateCalled[c.name])
 				require.Equal(t, expectCalls, expectationsCalled[c.name])
 			}
@@ -194,8 +192,7 @@ func TestDiffTester_Run_WithMemModifications(t *testing.T) {
 
 			// Validate that we invoked initState and setExpectations as expected
 			for _, c := range testCases {
-				// Difftester runs extra calls in order to analyze the tests
-				expectedCalls := len(versions) * (len(mods) + 2)
+				expectedCalls := len(versions) * (len(mods) + 1)
 				require.Equal(t, expectedCalls, initStateCalled[c.name])
 				require.Equal(t, expectedCalls, expectationsCalled[c.name])
 			}
@@ -252,8 +249,7 @@ func TestDiffTester_Run_WithPanic(t *testing.T) {
 
 			// Validate that we invoked initState and setExpectations as expected
 			for _, c := range testCases {
-				// Difftester runs extra calls in order to analyze the tests
-				expectedCalls := 2 * len(versions)
+				expectedCalls := len(versions)
 				require.Equal(t, expectedCalls, initStateCalled[c.name])
 				require.Equal(t, expectedCalls, expectationsCalled[c.name])
 			}
@@ -309,9 +305,8 @@ func TestDiffTester_Run_WithVm(t *testing.T) {
 
 	// Validate that we invoked initState and setExpectations as expected
 	for _, c := range testCases {
-		// Difftester runs extra calls in order to analyze the tests
-		require.Equal(t, 2, initStateCalled[c.name])
-		require.Equal(t, 2, expectationsCalled[c.name])
+		require.Equal(t, 1, initStateCalled[c.name])
+		require.Equal(t, 1, expectationsCalled[c.name])
 	}
 
 	// Validate that we ran the expected tests

cannon/mipsevm/tests/evm_common_test.go

@@ -891,7 +891,7 @@ func TestEVM_SysWriteHint(t *testing.T) {
 		return ExpectNormalExecution()
 	}
 
-	postCheck := func(t require.TestingT, tt testCase, vm VersionedVMTestCase, deps *TestDependencies) {
+	postCheck := func(t require.TestingT, tt testCase, vm VersionedVMTestCase, deps *TestDependencies, wit *mipsevm.StepWitness) {
 		trackingOracle, ok := deps.po.(*testutil.HintTrackingOracle)
 		require.True(t, ok)
 		require.Equal(t, tt.expectedHints, trackingOracle.Hints())