cannon: Fix GC emulation of Go programs

cannon/mipsevm/exec/memory.go

@@ -14,7 +14,10 @@ type MemoryTrackerImpl struct {
 	memory          *memory.Memory
 	lastMemAccess   uint32
 	memProofEnabled bool
-	memProof        [memory.MEM_PROOF_SIZE]byte
+	// proof of first unique memory access
+	memProof [memory.MEM_PROOF_SIZE]byte
+	// proof of second unique memory access
+	memProof2 [memory.MEM_PROOF_SIZE]byte
 }
 
 func NewMemoryTracker(memory *memory.Memory) *MemoryTrackerImpl {
@@ -31,6 +34,16 @@ func (m *MemoryTrackerImpl) TrackMemAccess(effAddr uint32) {
 	}
 }
 
+// TrackMemAccess2 creates a proof for a memory access following a call to TrackMemAccess
+// This is used to generate proofs for contiguous memory accesses within the same step
+func (m *MemoryTrackerImpl) TrackMemAccess2(effAddr uint32) {
+	if m.memProofEnabled && m.lastMemAccess+4 != effAddr {
+		panic(fmt.Errorf("unexpected disjointed mem access at %08x, last memory access is at %08x buffered", effAddr, m.lastMemAccess))
+	}
+	m.lastMemAccess = effAddr
+	m.memProof2 = m.memory.MerkleProof(effAddr)
+}
+
 func (m *MemoryTrackerImpl) Reset(enableProof bool) {
 	m.memProofEnabled = enableProof
 	m.lastMemAccess = ^uint32(0)
@@ -39,3 +52,7 @@ func (m *MemoryTrackerImpl) Reset(enableProof bool) {
 func (m *MemoryTrackerImpl) MemProof() [memory.MEM_PROOF_SIZE]byte {
 	return m.memProof
 }
+
+func (m *MemoryTrackerImpl) MemProof2() [memory.MEM_PROOF_SIZE]byte {
+	return m.memProof2
+}

cannon/mipsevm/exec/mips_syscalls.go

@@ -14,19 +14,21 @@ import (
 
 // Syscall codes
 const (
-	SysMmap       = 4090
-	SysBrk        = 4045
-	SysClone      = 4120
-	SysExitGroup  = 4246
-	SysRead       = 4003
-	SysWrite      = 4004
-	SysFcntl      = 4055
-	SysExit       = 4001
-	SysSchedYield = 4162
-	SysGetTID     = 4222
-	SysFutex      = 4238
-	SysOpen       = 4005
-	SysNanosleep  = 4166
+	SysMmap         = 4090
+	SysBrk          = 4045
+	SysClone        = 4120
+	SysExitGroup    = 4246
+	SysRead         = 4003
+	SysWrite        = 4004
+	SysFcntl        = 4055
+	SysExit         = 4001
+	SysSchedYield   = 4162
+	SysGetTID       = 4222
+	SysFutex        = 4238
+	SysOpen         = 4005
+	SysNanosleep    = 4166
+	SysClockGetTime = 4263
+	SysGetpid       = 4020
 )
 
 // Noop Syscall codes
@@ -67,7 +69,6 @@ const (
 	SysTimerCreate  = 4257
 	SysTimerSetTime = 4258
 	SysTimerDelete  = 4261
-	SysClockGetTime = 4263
 )
 
 // File descriptors
@@ -132,7 +133,21 @@ const (
 
 // Other constants
 const (
+	// SchedQuantum is the number of steps dedicated for a thread before it's preempted. Effectively used to emulate thread "time slices"
 	SchedQuantum = 100_000
+
+	// HZ is the assumed clock rate of an emulated MIPS32 CPU.
+	// The value of HZ is a rough estimate of the Cannon instruction count / second on a typical machine.
+	// HZ is used to emulate the clock_gettime syscall used by guest programs that have a Go runtime.
+	// The Go runtime consumes the system time to determine when to initiate gc assists and for goroutine scheduling.
+	// A HZ value that is too low (i.e. lower than the emulation speed) results in the main goroutine attempting to assist with GC more often.
+	// Adjust this value accordingly as the emulation speed changes. The HZ value should be within the same order of magnitude as the emulation speed.
+	HZ = 10_000_000
+
+	// ClockGettimeRealtimeFlag is the clock_gettime clock id for Linux's realtime clock: https://github.com/torvalds/linux/blob/ad618736883b8970f66af799e34007475fe33a68/include/uapi/linux/time.h#L49
+	ClockGettimeRealtimeFlag = 0
+	// ClockGettimeMonotonicFlag is the clock_gettime clock id for Linux's monotonic clock: https://github.com/torvalds/linux/blob/ad618736883b8970f66af799e34007475fe33a68/include/uapi/linux/time.h#L50
+	ClockGettimeMonotonicFlag = 1
 )
 
 func GetSyscallArgs(registers *[32]uint32) (syscallNum, a0, a1, a2, a3 uint32) {

cannon/mipsevm/memory/memory.go

@@ -328,6 +328,20 @@ func (m *Memory) Deserialize(in io.Reader) error {
 	return nil
 }
 
+func (m *Memory) Copy() *Memory {
+	out := NewMemory()
+	out.nodes = make(map[uint64]*[32]byte)
+	out.pages = make(map[uint32]*CachedPage)
+	out.lastPageKeys = [2]uint32{^uint32(0), ^uint32(0)}
+	out.lastPage = [2]*CachedPage{nil, nil}
+	for k, page := range m.pages {
+		data := new(Page)
+		*data = *page.Data
+		out.AllocPage(k).Data = data
+	}
+	return out
+}
+
 type memReader struct {
 	m     *Memory
 	addr  uint32

cannon/mipsevm/memory/memory_test.go

@@ -187,3 +187,11 @@ func TestMemoryJSON(t *testing.T) {
 	require.NoError(t, json.Unmarshal(dat, &res))
 	require.Equal(t, uint32(123), res.GetMemory(8))
 }
+
+func TestMemoryCopy(t *testing.T) {
+	m := NewMemory()
+	m.SetMemory(0x8000, 123)
+	mcpy := m.Copy()
+	require.Equal(t, uint32(123), mcpy.GetMemory(0x8000))
+	require.Equal(t, m.MerkleRoot(), mcpy.MerkleRoot())
+}

cannon/mipsevm/multithreaded/instrumented.go

@@ -73,7 +73,9 @@ func (m *InstrumentedState) Step(proof bool) (wit *mipsevm.StepWitness, err erro
 
 	if proof {
 		memProof := m.memoryTracker.MemProof()
+		memProof2 := m.memoryTracker.MemProof2()
 		wit.ProofData = append(wit.ProofData, memProof[:]...)
+		wit.ProofData = append(wit.ProofData, memProof2[:]...)
 		lastPreimageKey, lastPreimage, lastPreimageOffset := m.preimageOracle.LastPreimage()
 		if lastPreimageOffset != ^uint32(0) {
 			wit.PreimageOffset = lastPreimageOffset

cannon/mipsevm/multithreaded/instrumented_test.go

@@ -19,19 +19,23 @@ func vmFactory(state *State, po mipsevm.PreimageOracle, stdOut, stdErr io.Writer
 }
 
 func TestInstrumentedState_OpenMips(t *testing.T) {
+	t.Parallel()
 	// TODO(cp-903): Add mt-specific tests here
 	testutil.RunVMTests_OpenMips(t, CreateEmptyState, vmFactory, "clone.bin")
 }
 
 func TestInstrumentedState_Hello(t *testing.T) {
+	t.Parallel()
 	testutil.RunVMTest_Hello(t, CreateInitialState, vmFactory, false)
 }
 
 func TestInstrumentedState_Claim(t *testing.T) {
+	t.Parallel()
 	testutil.RunVMTest_Claim(t, CreateInitialState, vmFactory, false)
 }
 
 func TestInstrumentedState_MultithreadedProgram(t *testing.T) {
+	t.Parallel()
 	state, _ := testutil.LoadELFProgram(t, "../../testdata/example/bin/multithreaded.elf", CreateInitialState, false)
 	oracle := testutil.StaticOracle(t, []byte{})
 
@@ -54,26 +58,43 @@ func TestInstrumentedState_MultithreadedProgram(t *testing.T) {
 }
 
 func TestInstrumentedState_Alloc(t *testing.T) {
-	t.Skip("TODO(client-pod#906): Currently failing - need to debug.")
+	const MiB = 1024 * 1024
 
-	state, _ := testutil.LoadELFProgram(t, "../../testdata/example/bin/alloc.elf", CreateInitialState, false)
-	const numAllocs = 100 // where each alloc is a 32 MiB chunk
-	oracle := testutil.AllocOracle(t, numAllocs)
+	cases := []struct {
+		name                string
+		numAllocs           int
+		allocSize           int
+		maxMemoryUsageCheck int
+	}{
+		{name: "10 32MiB allocations", numAllocs: 10, allocSize: 32 * MiB, maxMemoryUsageCheck: 256 * MiB},
+		{name: "5 64MiB allocations", numAllocs: 5, allocSize: 64 * MiB, maxMemoryUsageCheck: 256 * MiB},
+		{name: "5 128MiB allocations", numAllocs: 5, allocSize: 128 * MiB, maxMemoryUsageCheck: 128 * 3 * MiB},
+	}
 
-	// completes in ~870 M steps
-	us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger(), nil)
-	for i := 0; i < 20_000_000_000; i++ {
-		if us.GetState().GetExited() {
-			break
-		}
-		_, err := us.Step(false)
-		require.NoError(t, err)
-		if state.Step%10_000_000 == 0 {
-			t.Logf("Completed %d steps", state.Step)
-		}
+	for _, test := range cases {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			state, _ := testutil.LoadELFProgram(t, "../../testdata/example/bin/alloc.elf", CreateInitialState, false)
+			oracle := testutil.AllocOracle(t, test.numAllocs, test.allocSize)
+
+			us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger(), nil)
+			// emulation shouldn't take more than 20 B steps
+			for i := 0; i < 20_000_000_000; i++ {
+				if us.GetState().GetExited() {
+					break
+				}
+				_, err := us.Step(false)
+				require.NoError(t, err)
+				if state.Step%10_000_000 == 0 {
+					t.Logf("Completed %d steps", state.Step)
+				}
+			}
+			memUsage := state.Memory.PageCount() * memory.PageSize
+			t.Logf("Completed in %d steps. cannon memory usage: %d KiB", state.Step, memUsage/1024/1024.0)
+			require.True(t, state.Exited, "must complete program")
+			require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
+			require.Less(t, memUsage, test.maxMemoryUsageCheck, "memory allocation is too large")
+		})
 	}
-	t.Logf("Completed in %d steps", state.Step)
-	require.True(t, state.Exited, "must complete program")
-	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
-	require.Less(t, state.Memory.PageCount()*memory.PageSize, 1*1024*1024*1024, "must not allocate more than 1 GiB")
 }

cannon/mipsevm/multithreaded/mips.go

@@ -143,6 +143,30 @@ func (m *InstrumentedState) handleSyscall() error {
 	case exec.SysOpen:
 		v0 = exec.SysErrorSignal
 		v1 = exec.MipsEBADF
+	case exec.SysClockGetTime:
+		switch a0 {
+		case exec.ClockGettimeRealtimeFlag, exec.ClockGettimeMonotonicFlag:
+			v0, v1 = 0, 0
+			var secs, nsecs uint32
+			if a0 == exec.ClockGettimeMonotonicFlag {
+				// monotonic clock_gettime is used by Go guest programs for goroutine scheduling and to implement
+				// `time.Sleep` (and other sleep related operations).
+				secs = uint32(m.state.Step / exec.HZ)
+				nsecs = uint32((m.state.Step % exec.HZ) * (1_000_000_000 / exec.HZ))
+			} // else realtime set to Unix Epoch
+
+			effAddr := a1 & 0xFFffFFfc
+			m.memoryTracker.TrackMemAccess(effAddr)
+			m.state.Memory.SetMemory(effAddr, secs)
+			m.memoryTracker.TrackMemAccess2(effAddr + 4)
+			m.state.Memory.SetMemory(effAddr+4, nsecs)
+		default:
+			v0 = exec.SysErrorSignal
+			v1 = exec.MipsEINVAL
+		}
+	case exec.SysGetpid:
+		v0 = 0
+		v1 = 0
 	case exec.SysMunmap:
 	case exec.SysGetAffinity:
 	case exec.SysMadvise:
@@ -173,7 +197,6 @@ func (m *InstrumentedState) handleSyscall() error {
 	case exec.SysTimerCreate:
 	case exec.SysTimerSetTime:
 	case exec.SysTimerDelete:
-	case exec.SysClockGetTime:
 	default:
 		m.Traceback()
 		panic(fmt.Sprintf("unrecognized syscall: %d", syscallNum))

cannon/mipsevm/multithreaded/testutil/expectations.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/stretchr/testify/require"
 
+	"github.com/ethereum-optimism/optimism/cannon/mipsevm/memory"
 	"github.com/ethereum-optimism/optimism/cannon/mipsevm/multithreaded"
 )
 
@@ -33,6 +34,7 @@ type ExpectedMTState struct {
 	prestateActiveThreadOrig    ExpectedThreadState // Cached for internal use
 	ActiveThreadId              uint32
 	threadExpectations          map[uint32]*ExpectedThreadState
+	memoryExpectations          *memory.Memory
 }
 
 type ExpectedThreadState struct {
@@ -81,6 +83,7 @@ func NewExpectedMTState(fromState *multithreaded.State) *ExpectedMTState {
 		prestateActiveThreadOrig: *newExpectedThreadState(currentThread), // Cache prestate thread for internal use
 		ActiveThreadId:           currentThread.ThreadId,
 		threadExpectations:       expectedThreads,
+		memoryExpectations:       fromState.Memory.Copy(),
 	}
 }
 
@@ -109,6 +112,17 @@ func (e *ExpectedMTState) ExpectStep() {
 	e.StepsSinceLastContextSwitch += 1
 }
 
+func (e *ExpectedMTState) ExpectMemoryWrite(addr uint32, val uint32) {
+	e.memoryExpectations.SetMemory(addr, val)
+	e.MemoryRoot = e.memoryExpectations.MerkleRoot()
+}
+
+func (e *ExpectedMTState) ExpectMemoryWriteMultiple(addr uint32, val uint32, addr2 uint32, val2 uint32) {
+	e.memoryExpectations.SetMemory(addr, val)
+	e.memoryExpectations.SetMemory(addr2, val)
+	e.MemoryRoot = e.memoryExpectations.MerkleRoot()
+}
+
 func (e *ExpectedMTState) ExpectPreemption(preState *multithreaded.State) {
 	e.ActiveThreadId = FindNextThread(preState).ThreadId
 	e.StepsSinceLastContextSwitch = 0

cannon/mipsevm/tests/evm_common_test.go

@@ -488,8 +488,8 @@ func TestHelloEVM(t *testing.T) {
 				// verify the post-state matches.
 				// TODO: maybe more readable to decode the evmPost state, and do attribute-wise comparison.
 				goPost, _ := goVm.GetState().EncodeWitness()
-				require.Equal(t, hexutil.Bytes(goPost).String(), hexutil.Bytes(evmPost).String(),
-					"mipsevm produced different state than EVM")
+				require.Equalf(t, hexutil.Bytes(goPost).String(), hexutil.Bytes(evmPost).String(),
+					"mipsevm produced different state than EVM. insn: %x", insn)
 			}
 			end := time.Now()
 			delta := end.Sub(start)