diff --git a/.circleci/config.yml b/.circleci/config.yml index 8e315c9fbd0bc..981dcac34c769 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2229,7 +2229,7 @@ workflows: notify: true mentions: "@proofs-team" no_output_timeout: 90m - test_timeout: 240m + test_timeout: 480m resource_class: ethereum-optimism/latitude-fps-1 context: - slack diff --git a/cannon/cmd/run.go b/cannon/cmd/run.go index 0cad2107b1730..5f6c2c685a098 100644 --- a/cannon/cmd/run.go +++ b/cannon/cmd/run.go @@ -395,7 +395,7 @@ func Run(ctx *cli.Context) error { } } - state, err := versions.LoadStateFromFile(ctx.Path(RunInputFlag.Name)) + state, err := versions.LoadStateFromFileWithLargeICache(ctx.Path(RunInputFlag.Name)) if err != nil { return fmt.Errorf("failed to load state: %w", err) } diff --git a/cannon/mipsevm/memory/memory.go b/cannon/mipsevm/memory/memory.go index f3ec580b851b2..159e0b3ebf321 100644 --- a/cannon/mipsevm/memory/memory.go +++ b/cannon/mipsevm/memory/memory.go @@ -66,20 +66,16 @@ type PageIndex interface { New(pages map[Word]*CachedPage) PageIndex } -func NewMemory(sizes ...arch.Word) *Memory { - codeSize := arch.Word(1 << 31) // Default: 2 GiB code - heapSize := arch.Word(1 << 31) // Default: 2 GiB heap - - if len(sizes) > 0 && sizes[0] != 0 { - codeSize = sizes[0] - } - if len(sizes) > 1 && sizes[1] != 0 { - heapSize = sizes[1] - } - +func NewMemoryWithLargeRegions() *Memory { + const codeSize = 1 << 31 + const heapSize = 1 << 31 return NewBinaryTreeMemory(codeSize, heapSize) } +func NewMemory() *Memory { + return NewBinaryTreeMemory(4096, 4096) +} + // start end size gap func (m *Memory) GetAllocatedRanges() [][4]Word { var ranges [][4]Word diff --git a/cannon/mipsevm/multithreaded/state.go b/cannon/mipsevm/multithreaded/state.go index 35ca34d1bc077..f2405f7c1bdce 100644 --- a/cannon/mipsevm/multithreaded/state.go +++ b/cannon/mipsevm/multithreaded/state.go @@ -70,6 +70,8 @@ type State struct { // LastHint is optional metadata, and not part of the VM state itself. LastHint hexutil.Bytes + + UseLargeICache bool } var _ mipsevm.FPVMState = (*State)(nil) @@ -333,7 +335,11 @@ func (s *State) Serialize(out io.Writer) error { func (s *State) Deserialize(in io.Reader) error { bin := serialize.NewBinaryReader(in) - s.Memory = memory.NewMemory() + if s.UseLargeICache { + s.Memory = memory.NewMemoryWithLargeRegions() + } else { + s.Memory = memory.NewMemory() + } if err := s.Memory.Deserialize(in); err != nil { return err } diff --git a/cannon/mipsevm/multithreaded/testutil/state.go b/cannon/mipsevm/multithreaded/testutil/state.go index 089aa2e06b21e..06e5c4581436f 100644 --- a/cannon/mipsevm/multithreaded/testutil/state.go +++ b/cannon/mipsevm/multithreaded/testutil/state.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum-optimism/optimism/cannon/mipsevm" "github.com/ethereum-optimism/optimism/cannon/mipsevm/arch" "github.com/ethereum-optimism/optimism/cannon/mipsevm/exec" - "github.com/ethereum-optimism/optimism/cannon/mipsevm/memory" "github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded" "github.com/ethereum-optimism/optimism/cannon/mipsevm/testutil" ) @@ -202,9 +201,3 @@ func RandomState(seed int) *multithreaded.State { mut.Randomize(int64(seed)) return state } - -func WithRegionSize(codeSize arch.Word, heapSize arch.Word) StateOption { - return func(state *StateMutator) { - state.state.Memory = memory.NewMemory(codeSize, heapSize) - } -} diff --git a/cannon/mipsevm/tests/difftester_test.go b/cannon/mipsevm/tests/difftester_test.go index 26a81413d4ffa..2647689bf79c9 100644 --- a/cannon/mipsevm/tests/difftester_test.go +++ b/cannon/mipsevm/tests/difftester_test.go @@ -48,7 +48,7 @@ func TestDiffTester_Run_SimpleTest(t *testing.T) { // Run tests tRunner := newMockTestRunner(t) NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). run(tRunner, testCases) @@ -109,7 +109,7 @@ func TestDiffTester_Run_WithSteps(t *testing.T) { // Run tests tRunner := newMockTestRunner(t) NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). run(tRunner, cases, WithSteps(oc.steps)) @@ -187,7 +187,7 @@ func TestDiffTester_Run_WithMemModifications(t *testing.T) { tRunner := newMockTestRunner(t) NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). run(tRunner, testCases, opts...) @@ -244,7 +244,7 @@ func TestDiffTester_Run_WithPanic(t *testing.T) { // Run tests tRunner := newMockTestRunner(t) NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). run(tRunner, testCases) @@ -300,7 +300,7 @@ func TestDiffTester_Run_WithVm(t *testing.T) { // Run tests tRunner := newMockTestRunner(t) NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). run(tRunner, testCases, WithVm(vm)) diff --git a/cannon/mipsevm/tests/evm_common64_test.go b/cannon/mipsevm/tests/evm_common64_test.go index 712301a3d7772..0922995d236b2 100644 --- a/cannon/mipsevm/tests/evm_common64_test.go +++ b/cannon/mipsevm/tests/evm_common64_test.go @@ -171,7 +171,7 @@ func TestEVM_SingleStep_Shift64(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -567,7 +567,7 @@ func TestEVM_SingleStep_DCloDClz64(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } diff --git a/cannon/mipsevm/tests/evm_common_test.go b/cannon/mipsevm/tests/evm_common_test.go index c59b2d9ac7d40..46065708b1231 100644 --- a/cannon/mipsevm/tests/evm_common_test.go +++ b/cannon/mipsevm/tests/evm_common_test.go @@ -24,9 +24,6 @@ import ( "github.com/ethereum-optimism/optimism/cannon/mipsevm/versions" ) -const testCodeRegionSize = 4096 -const testHeapSize = 4096 - type insnCache interface { UpdateInstructionCache(pc arch.Word) } @@ -75,7 +72,7 @@ func TestEVM_SingleStep_Jump(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -178,7 +175,7 @@ func TestEVM_SingleStep_Lui(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -222,7 +219,7 @@ func TestEVM_SingleStep_CloClz(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -271,7 +268,7 @@ func TestEVM_SingleStep_MovzMovn(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -309,7 +306,7 @@ func TestEVM_SingleStep_MfhiMflo(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -384,7 +381,7 @@ func TestEVM_SingleStep_MthiMtlo(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -429,7 +426,7 @@ func TestEVM_SingleStep_BeqBne(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -492,7 +489,7 @@ func TestEVM_SingleStep_SlSr(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -543,7 +540,7 @@ func TestEVM_SingleStep_JrJalr(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -572,7 +569,7 @@ func TestEVM_SingleStep_Sync(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -630,7 +627,7 @@ func TestEVM_MMap(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -721,7 +718,7 @@ func TestEVM_SysGetRandom(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithStep(step), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithStep(step)). SetExpectations(setExpectations). Run(t, cases, SkipAutomaticMemoryReservationTests()) //Was getting failure from the “automatic memory reservation” modifier that the DiffTester adds. @@ -915,7 +912,7 @@ func TestEVM_SysWriteHint(t *testing.T) { po := func() mipsevm.PreimageOracle { return &testutil.HintTrackingOracle{} } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). PostCheck(postCheck). Run(t, cases, WithPreimageOracle(po)) @@ -970,7 +967,7 @@ func TestEVM_Fault(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } diff --git a/cannon/mipsevm/tests/evm_multithreaded64_test.go b/cannon/mipsevm/tests/evm_multithreaded64_test.go index bcf5b7ba6d6cf..b04b397a105bc 100644 --- a/cannon/mipsevm/tests/evm_multithreaded64_test.go +++ b/cannon/mipsevm/tests/evm_multithreaded64_test.go @@ -95,7 +95,7 @@ func TestEVM_MT64_LL(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(0x40), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0x40)). SetExpectations(setExpectations). Run(t, cases) } @@ -200,7 +200,7 @@ func TestEVM_MT64_SC(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, SkipAutomaticMemoryReservationTests()) } @@ -279,7 +279,7 @@ func TestEVM_MT64_LLD(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(0x40), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0x40)). SetExpectations(setExpectations). Run(t, cases) } @@ -385,7 +385,7 @@ func TestEVM_MT64_SCD(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, SkipAutomaticMemoryReservationTests()) } @@ -477,7 +477,7 @@ func TestEVM_MT_SysRead_Preimage64(t *testing.T) { po := func() mipsevm.PreimageOracle { return testutil.StaticOracle(t, preimageValue) } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, WithPreimageOracle(po)) } @@ -518,7 +518,7 @@ func TestEVM_MT_SysReadWrite_WithEventFd(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -581,7 +581,7 @@ func TestEVM_MT_StoreOpsClearMemReservation64(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -702,7 +702,7 @@ func TestEVM_UndefinedSyscall(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } diff --git a/cannon/mipsevm/tests/evm_multithreaded_test.go b/cannon/mipsevm/tests/evm_multithreaded_test.go index 1c534e7533a38..1f455fc63d474 100644 --- a/cannon/mipsevm/tests/evm_multithreaded_test.go +++ b/cannon/mipsevm/tests/evm_multithreaded_test.go @@ -92,7 +92,7 @@ func TestEVM_MT_LL(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithPCAndNextPC(0x40), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0x40)). SetExpectations(setExpectations). Run(t, cases) } @@ -192,7 +192,7 @@ func TestEVM_MT_SC(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, SkipAutomaticMemoryReservationTests()) } @@ -244,7 +244,7 @@ func TestEVM_SysClone_FlagHandling(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -281,7 +281,7 @@ func TestEVM_SysClone_Successful(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -340,7 +340,7 @@ func TestEVM_SysGetTID(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -395,7 +395,7 @@ func TestEVM_SysExit(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -435,7 +435,7 @@ func TestEVM_PopExitedThread(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -498,7 +498,7 @@ func TestEVM_SysFutex_WaitPrivate(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -552,7 +552,7 @@ func TestEVM_SysFutex_WakePrivate(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -626,7 +626,7 @@ func TestEVM_SysFutex_UnsupportedOp(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -683,7 +683,7 @@ func runPreemptSyscall(t *testing.T, syscallName string, syscallNum uint32) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -702,7 +702,7 @@ func TestEVM_SysOpen(t *testing.T) { } NewSimpleDiffTester(). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t) } @@ -721,7 +721,7 @@ func TestEVM_SysGetPID(t *testing.T) { } NewSimpleDiffTester(). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t) } @@ -830,7 +830,7 @@ func testEVM_SysClockGettime(t *testing.T, clkid Word) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, SkipAutomaticMemoryReservationTests()) } @@ -852,7 +852,7 @@ func TestEVM_SysClockGettimeNonMonotonic(t *testing.T) { } NewSimpleDiffTester(). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t) } @@ -893,7 +893,7 @@ func TestEVM_EmptyThreadStacks(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -965,7 +965,7 @@ func TestEVM_NormalTraversal_Full(t *testing.T) { } diffTester := NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) for _, bt := range baseTests { @@ -1016,7 +1016,7 @@ func TestEVM_SchedQuantumThreshold(t *testing.T) { } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } diff --git a/cannon/mipsevm/tests/fuzz_evm_common64_test.go b/cannon/mipsevm/tests/fuzz_evm_common64_test.go index e00b6b741a282..6518a46aa3239 100644 --- a/cannon/mipsevm/tests/fuzz_evm_common64_test.go +++ b/cannon/mipsevm/tests/fuzz_evm_common64_test.go @@ -89,7 +89,7 @@ func mulOpCheck(f *testing.F, multiplier multiplierFn, opcode uint32, expectRdRe } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithPCAndNextPC(0), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0)). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, rs, rt, seed int64) { diff --git a/cannon/mipsevm/tests/fuzz_evm_common_test.go b/cannon/mipsevm/tests/fuzz_evm_common_test.go index 626069be6abe5..b475f9508b8e1 100644 --- a/cannon/mipsevm/tests/fuzz_evm_common_test.go +++ b/cannon/mipsevm/tests/fuzz_evm_common_test.go @@ -26,7 +26,7 @@ func FuzzStateSyscallBrk(f *testing.F) { initState := func(t require.TestingT, state *multithreaded.State, vm VersionedVMTestCase, r *testutil.RandHelper, goVm mipsevm.FPVM) { state.GetRegistersRef()[2] = arch.SysBrk - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) } setExpectations := func(t require.TestingT, expected *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult { @@ -37,7 +37,7 @@ func FuzzStateSyscallBrk(f *testing.F) { } diffTester := NewSimpleDiffTester(). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, seed int64) { @@ -64,7 +64,7 @@ func FuzzStateSyscallMmap(f *testing.F) { state.GetRegistersRef()[2] = arch.SysMmap state.GetRegistersRef()[4] = c.addr state.GetRegistersRef()[5] = c.siz - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) } setExpectations := func(t require.TestingT, c testCase, expected *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult { @@ -91,7 +91,7 @@ func FuzzStateSyscallMmap(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, addr Word, siz Word, heap Word, seed int64) { @@ -109,7 +109,7 @@ func FuzzStateSyscallExitGroup(f *testing.F) { initState := func(t require.TestingT, c testCase, state *multithreaded.State, vm VersionedVMTestCase, r *testutil.RandHelper, goVm mipsevm.FPVM) { state.GetRegistersRef()[2] = arch.SysExitGroup state.GetRegistersRef()[4] = Word(c.exitCode) - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) } setExpectations := func(t require.TestingT, c testCase, expected *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult { @@ -121,7 +121,7 @@ func FuzzStateSyscallExitGroup(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, exitCode uint8, seed int64) { @@ -141,7 +141,7 @@ func FuzzStateSyscallFcntl(f *testing.F) { state.GetRegistersRef()[2] = arch.SysFcntl state.GetRegistersRef()[4] = c.fd state.GetRegistersRef()[5] = c.cmd - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) } setExpectations := func(t require.TestingT, c testCase, expected *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult { @@ -176,7 +176,7 @@ func FuzzStateSyscallFcntl(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, fd Word, cmd Word, seed int64) { @@ -200,7 +200,7 @@ func FuzzStateHintRead(f *testing.F) { state.GetRegistersRef()[4] = exec.FdHintRead state.GetRegistersRef()[5] = c.addr state.GetRegistersRef()[6] = c.count - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) } setExpectations := func(t require.TestingT, c testCase, expected *mtutil.ExpectedState, vm VersionedVMTestCase) ExpectedExecResult { @@ -215,7 +215,7 @@ func FuzzStateHintRead(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). PostCheck(postCheck) @@ -251,7 +251,7 @@ func FuzzStatePreimageRead(f *testing.F) { state.GetRegistersRef()[4] = exec.FdPreimageRead state.GetRegistersRef()[5] = c.addr state.GetRegistersRef()[6] = c.count - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) state.GetMemory().SetWord(testutil.EffAddr(c.addr), preexistingMemoryVal) } @@ -286,7 +286,7 @@ func FuzzStatePreimageRead(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). PostCheck(postCheck) @@ -368,7 +368,7 @@ func FuzzStateHintWrite(f *testing.F) { state.GetRegistersRef()[4] = exec.FdHintWrite state.GetRegistersRef()[5] = c.addr state.GetRegistersRef()[6] = c.count - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) err := state.GetMemory().SetMemoryRange(c.addr, bytes.NewReader(c.hintData[int(len(c.lastHint)):])) require.NoError(t, err) } @@ -389,7 +389,7 @@ func FuzzStateHintWrite(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[*testCase]). - InitState(initState, mtutil.WithPCAndNextPC(0), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0)). SetExpectations(setExpectations). PostCheck(postCheck) @@ -431,7 +431,7 @@ func FuzzStatePreimageWrite(f *testing.F) { state.GetRegistersRef()[4] = exec.FdPreimageWrite state.GetRegistersRef()[5] = c.addr state.GetRegistersRef()[6] = c.count - testutil.StoreInstruction(state.GetMemory(), state.GetPC(), syscallInsn) + storeInsnWithCache(state, goVm, state.GetPC(), syscallInsn) state.GetMemory().SetWord(testutil.EffAddr(c.addr), arch.ByteOrderWord.Word(preexistingMemoryVal[:])) } @@ -462,7 +462,7 @@ func FuzzStatePreimageWrite(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithPCAndNextPC(0), mtutil.WithPreimageKey(preimageKey), mtutil.WithPreimageOffset(128), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(0), mtutil.WithPreimageKey(preimageKey), mtutil.WithPreimageOffset(128)). SetExpectations(setExpectations). PostCheck(postCheck) diff --git a/cannon/mipsevm/tests/fuzz_evm_multithreaded_test.go b/cannon/mipsevm/tests/fuzz_evm_multithreaded_test.go index 8d83c89fb2884..56933430c62f9 100644 --- a/cannon/mipsevm/tests/fuzz_evm_multithreaded_test.go +++ b/cannon/mipsevm/tests/fuzz_evm_multithreaded_test.go @@ -58,7 +58,7 @@ func FuzzStateSyscallCloneMT(f *testing.F) { } diffTester := NewDiffTester(NoopTestNamer[testCase]). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations) f.Fuzz(func(t *testing.T, nextThreadId, stackPtr Word, seed int64) { diff --git a/cannon/mipsevm/tests/testfuncs_test.go b/cannon/mipsevm/tests/testfuncs_test.go index ab4e8f09cad34..797f1f8919bad 100644 --- a/cannon/mipsevm/tests/testfuncs_test.go +++ b/cannon/mipsevm/tests/testfuncs_test.go @@ -70,7 +70,7 @@ func testOperators(t *testing.T, testCases []operatorTestCase, mips32Insn bool) } NewDiffTester((operatorTestCase).Name). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -134,7 +134,7 @@ func testMulDiv(t *testing.T, templateCases []mulDivTestCase, mips32Insn bool) { } NewDiffTester((mulDivTestCase).Name). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -184,7 +184,7 @@ func testLoadStore(t *testing.T, cases []loadStoreTestCase) { } NewDiffTester((loadStoreTestCase).Name). - InitState(initState, mtutil.WithPCAndNextPC(pc), mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState, mtutil.WithPCAndNextPC(pc)). SetExpectations(setExpectations). Run(t, cases) } @@ -226,7 +226,7 @@ func testBranch(t *testing.T, cases []branchTestCase) { } NewDiffTester((branchTestCase).Name). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases) } @@ -260,7 +260,7 @@ func testNoopSyscall(t *testing.T, vm VersionedVMTestCase, syscalls map[string]u } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, WithVm(vm)) } @@ -292,7 +292,7 @@ func testUnsupportedSyscall(t *testing.T, vm VersionedVMTestCase, unsupportedSys } NewDiffTester(testNamer). - InitState(initState, mtutil.WithRegionSize(testCodeRegionSize, testHeapSize)). + InitState(initState). SetExpectations(setExpectations). Run(t, cases, WithVm(vm)) } diff --git a/cannon/mipsevm/versions/state.go b/cannon/mipsevm/versions/state.go index 3c02599e2f9a8..c63e30b8d3abd 100644 --- a/cannon/mipsevm/versions/state.go +++ b/cannon/mipsevm/versions/state.go @@ -27,6 +27,14 @@ func LoadStateFromFile(path string) (*VersionedState, error) { return serialize.LoadSerializedBinary[VersionedState](path) } +func LoadStateFromFileWithLargeICache(path string) (*VersionedStateWithLargeICache, error) { + if !serialize.IsBinaryFile(path) { + // JSON states are always singlethreaded v1 which is no longer supported + return nil, fmt.Errorf("%w: %s", ErrUnsupportedVersion, VersionSingleThreaded) + } + return serialize.LoadSerializedBinary[VersionedStateWithLargeICache](path) +} + func NewFromState(vers StateVersion, state mipsevm.FPVMState) (*VersionedState, error) { switch state := state.(type) { case *multithreaded.State: @@ -106,3 +114,29 @@ func (s *VersionedState) Deserialize(in io.Reader) error { func (s *VersionedState) MarshalJSON() ([]byte, error) { return nil, fmt.Errorf("%w for type %T", ErrJsonNotSupported, s.FPVMState) } + +// VersionedStateWithLargeICache is a VersionedState that allocates a large memory region for the i-cache. +type VersionedStateWithLargeICache struct { + VersionedState +} + +func (s *VersionedStateWithLargeICache) Deserialize(in io.Reader) error { + bin := serialize.NewBinaryReader(in) + if err := bin.ReadUInt(&s.Version); err != nil { + return err + } + + if IsSupportedMultiThreaded64(s.Version) { + if arch.IsMips32 { + return ErrUnsupportedMipsArch + } + state := &multithreaded.State{UseLargeICache: true} + if err := state.Deserialize(in); err != nil { + return err + } + s.FPVMState = state + return nil + } else { + return fmt.Errorf("%w: %d", ErrUnknownVersion, s.Version) + } +} diff --git a/op-e2e/faultproofs/util.go b/op-e2e/faultproofs/util.go index 2654ef5b05025..7a42d45ed547d 100644 --- a/op-e2e/faultproofs/util.go +++ b/op-e2e/faultproofs/util.go @@ -3,6 +3,9 @@ package faultproofs import ( "crypto/ecdsa" "fmt" + "os" + "strconv" + "sync" "testing" op_e2e "github.com/ethereum-optimism/optimism/op-e2e" @@ -190,8 +193,49 @@ func RunTestsAcrossVmTypes[T any](t *testing.T, testCases []T, test VMTestCase[T testName := options.testNameModifier(string(allocType), testCase) t.Run(testName, func(t *testing.T) { op_e2e.InitParallel(t, op_e2e.UsesCannon) - test(t, allocType, testCase) + func() { + limiter.Acquire() + defer limiter.Release() + test(t, allocType, testCase) + }() }) } } } + +var executorLimitEnv = os.Getenv("OP_E2E_EXECUTOR_LIMIT") + +type executorLimiter struct { + ch chan struct{} +} + +func (l *executorLimiter) Acquire() { + // TODO: sample memory usage over time to admit more tests and reduce total runtime. + initExecutorLimiter() + l.ch <- struct{}{} +} + +func (l *executorLimiter) Release() { + <-l.ch +} + +var limiter executorLimiter +var limiterOnce sync.Once + +func initExecutorLimiter() { + limiterOnce.Do(func() { + var executorLimit uint64 + if executorLimitEnv != "" { + var err error + executorLimit, err = strconv.ParseUint(executorLimitEnv, 10, 0) + if err != nil { + panic(fmt.Sprintf("Could not parse OP_E2E_EXECUTOR_LIMIT env var %v: %v", executorLimitEnv, err)) + } + } else { + // faultproof tests may use 6 GiB of memory. So let's be very conservative and aggressively limit the number of test executions + // considering other processes running on the same machine. + executorLimit = 8 + } + limiter = executorLimiter{ch: make(chan struct{}, executorLimit)} + }) +}