cannon: Fix GC emulation of Go programs

Improves Linux/MIPS32 emulation for Go programs that utilize the garbage
collector and goroutine scheduling.

This adds support for the following syscalls:

- getpid - used by the go scheduler
- clock_gettime - used by the go scheduler and for GC assists and to properly emulate
  time related operations such as `time.Sleep`.

Note on GC assists:

The Go GC relies on `clock_gettime` for GC "assists", whereby a mutator can perform a little bit
of GC without waiting for the scheduler to do so.
A monotonic clock (runtime.nanotime) is used to compute the current goroutine's compute budget.
By modeling a MIPS32 CPU that runs at some clock speed (ex: 10 MHz), we can provide a consistent
emulation of monotonic time needed by the Go runtime.
All other clock_gettime flags are handled as unimplemented syscalls.

cannon/mipsevm/exec/memory.go

@@ -14,7 +14,10 @@ type MemoryTrackerImpl struct {
 	memory          *memory.Memory
 	lastMemAccess   uint32
 	memProofEnabled bool
-	memProof        [memory.MEM_PROOF_SIZE]byte
+	// proof of first unique memory access
+	memProof [memory.MEM_PROOF_SIZE]byte
+	// proof of second unique memory access
+	memProof2 [memory.MEM_PROOF_SIZE]byte
 }
 
 func NewMemoryTracker(memory *memory.Memory) *MemoryTrackerImpl {
@@ -31,6 +34,16 @@ func (m *MemoryTrackerImpl) TrackMemAccess(effAddr uint32) {
 	}
 }
 
+// TrackMemAccess2 creates a proof for a memory access following a call to TrackMemAccess
+// This is used to generate proofs for contiguous memory accesses within the same step
+func (m *MemoryTrackerImpl) TrackMemAccess2(effAddr uint32) {
+	if m.memProofEnabled && m.lastMemAccess+4 != effAddr {
+		panic(fmt.Errorf("unexpected disjointed mem access at %08x, last memory access is at %08x buffered", effAddr, m.lastMemAccess))
+	}
+	m.lastMemAccess = effAddr
+	m.memProof2 = m.memory.MerkleProof(effAddr)
+}
+
 func (m *MemoryTrackerImpl) Reset(enableProof bool) {
 	m.memProofEnabled = enableProof
 	m.lastMemAccess = ^uint32(0)
@@ -39,3 +52,7 @@ func (m *MemoryTrackerImpl) Reset(enableProof bool) {
 func (m *MemoryTrackerImpl) MemProof() [memory.MEM_PROOF_SIZE]byte {
 	return m.memProof
 }
+
+func (m *MemoryTrackerImpl) MemProof2() [memory.MEM_PROOF_SIZE]byte {
+	return m.memProof2
+}

cannon/mipsevm/exec/mips_syscalls.go

@@ -14,19 +14,21 @@ import (
 
 // Syscall codes
 const (
-	SysMmap       = 4090
-	SysBrk        = 4045
-	SysClone      = 4120
-	SysExitGroup  = 4246
-	SysRead       = 4003
-	SysWrite      = 4004
-	SysFcntl      = 4055
-	SysExit       = 4001
-	SysSchedYield = 4162
-	SysGetTID     = 4222
-	SysFutex      = 4238
-	SysOpen       = 4005
-	SysNanosleep  = 4166
+	SysMmap         = 4090
+	SysBrk          = 4045
+	SysClone        = 4120
+	SysExitGroup    = 4246
+	SysRead         = 4003
+	SysWrite        = 4004
+	SysFcntl        = 4055
+	SysExit         = 4001
+	SysSchedYield   = 4162
+	SysGetTID       = 4222
+	SysFutex        = 4238
+	SysOpen         = 4005
+	SysNanosleep    = 4166
+	SysClockGetTime = 4263
+	SysGetpid       = 4020
 )
 
 // Noop Syscall codes
@@ -67,7 +69,6 @@ const (
 	SysTimerCreate  = 4257
 	SysTimerSetTime = 4258
 	SysTimerDelete  = 4261
-	SysClockGetTime = 4263
 )
 
 // File descriptors
@@ -132,7 +133,18 @@ const (
 
 // Other constants
 const (
+	// SchedQuantum is the number of steps dedicated for a thread before it's preempted. Effectively used to emulate thread "time slices"
 	SchedQuantum = 100_000
+
+	// HZ is the assumed clock rate of an emulated MIPS32 CPU.
+	// The value of HZ is a rough estimate of the Cannon instruction count / second on a typical machine.
+	// HZ is used to emulate the clock_gettime syscall used by guest programs that have a Go runtime.
+	// The Go runtime consumes the system time to determine when to initiate gc assists and for goroutine scheduling.
+	// A HZ value that is too low (i.e. lower than the emulation speed) results in the main goroutine attempting to assist with GC more often.
+	// Adjust this value accordingly as the emulation speed changes. The HZ value should be within the same order of magnitude as the emulation speed.
+	HZ = 10_000_000
+
+	ClockGettimeMonotonicFlag = 1
 )
 
 func GetSyscallArgs(registers *[32]uint32) (syscallNum, a0, a1, a2, a3 uint32) {

cannon/mipsevm/multithreaded/instrumented.go

@@ -83,7 +83,9 @@ func (m *InstrumentedState) Step(proof bool) (wit *mipsevm.StepWitness, err erro
 
 	if proof {
 		memProof := m.memoryTracker.MemProof()
+		memProof2 := m.memoryTracker.MemProof2()
 		wit.ProofData = append(wit.ProofData, memProof[:]...)
+		wit.ProofData = append(wit.ProofData, memProof2[:]...)
 		lastPreimageKey, lastPreimage, lastPreimageOffset := m.preimageOracle.LastPreimage()
 		if lastPreimageOffset != ^uint32(0) {
 			wit.PreimageOffset = lastPreimageOffset

cannon/mipsevm/multithreaded/instrumented_test.go

@@ -54,26 +54,42 @@ func TestInstrumentedState_MultithreadedProgram(t *testing.T) {
 }
 
 func TestInstrumentedState_Alloc(t *testing.T) {
-	t.Skip("TODO(client-pod#906): Currently failing - need to debug.")
+	const maxMemoryUsageCheck = 512 * 1024 * 1024 // 512 MiB
 
-	state, _ := testutil.LoadELFProgram(t, "../../testdata/example/bin/alloc.elf", CreateInitialState, false)
-	const numAllocs = 100 // where each alloc is a 32 MiB chunk
-	oracle := testutil.AllocOracle(t, numAllocs)
+	cases := []struct {
+		name      string
+		numAllocs int
+		allocSize int
+	}{
+		{name: "100 32MiB allocations", numAllocs: 100, allocSize: 32 * 1024 * 1024},
+		{name: "50 64MiB allocations", numAllocs: 50, allocSize: 64 * 1024 * 1024},
+		{name: "10 128MiB allocations", numAllocs: 10, allocSize: 128 * 1024 * 1024},
+	}
 
-	// completes in ~870 M steps
-	us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger())
-	for i := 0; i < 20_000_000_000; i++ {
-		if us.GetState().GetExited() {
-			break
-		}
-		_, err := us.Step(false)
-		require.NoError(t, err)
-		if state.Step%10_000_000 == 0 {
-			t.Logf("Completed %d steps", state.Step)
-		}
+	for _, test := range cases {
+		test := test
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			state, _ := testutil.LoadELFProgram(t, "../../testdata/example/bin/alloc.elf", CreateInitialState, false)
+			oracle := testutil.AllocOracle(t, test.numAllocs, test.allocSize)
+
+			us := NewInstrumentedState(state, oracle, os.Stdout, os.Stderr, testutil.CreateLogger())
+			// emulation shouldn't take more than 20 B steps
+			for i := 0; i < 20_000_000_000; i++ {
+				if us.GetState().GetExited() {
+					break
+				}
+				_, err := us.Step(false)
+				require.NoError(t, err)
+				if state.Step%10_000_000 == 0 {
+					t.Logf("Completed %d steps", state.Step)
+				}
+			}
+			memUsage := state.Memory.PageCount() * memory.PageSize
+			t.Logf("Completed in %d steps. cannon memory usage: %d KiB", state.Step, memUsage/1024/1024.0)
+			require.True(t, state.Exited, "must complete program")
+			require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
+			require.Less(t, memUsage, maxMemoryUsageCheck, "memory allocation is too large")
+		})
 	}
-	t.Logf("Completed in %d steps", state.Step)
-	require.True(t, state.Exited, "must complete program")
-	require.Equal(t, uint8(0), state.ExitCode, "exit with 0")
-	require.Less(t, state.Memory.PageCount()*memory.PageSize, 1*1024*1024*1024, "must not allocate more than 1 GiB")
 }

cannon/mipsevm/multithreaded/mips.go

@@ -143,6 +143,22 @@ func (m *InstrumentedState) handleSyscall() error {
 	case exec.SysOpen:
 		v0 = exec.SysErrorSignal
 		v1 = exec.MipsEBADF
+	case exec.SysClockGetTime:
+		v0 = 0
+		v1 = 0
+		// clock_gettime is used by Go guest programs for goroutine scheduling and to implement `time.Sleep` (and other sleep related operations).
+		if a0 == exec.ClockGettimeMonotonicFlag {
+			secs := uint32(m.state.Step / exec.HZ)
+			nsecs := uint32((m.state.Step % exec.HZ) * (1_000_000_000 / exec.HZ))
+			effAddr := a1 & 0xFFffFFfc
+			m.memoryTracker.TrackMemAccess(effAddr)
+			m.state.Memory.SetMemory(effAddr, secs)
+			m.memoryTracker.TrackMemAccess2(effAddr + 4)
+			m.state.Memory.SetMemory(effAddr+4, nsecs)
+		}
+	case exec.SysGetpid:
+		v0 = 0
+		v1 = 0
 	case exec.SysMunmap:
 	case exec.SysGetAffinity:
 	case exec.SysMadvise:
@@ -173,7 +189,6 @@ func (m *InstrumentedState) handleSyscall() error {
 	case exec.SysTimerCreate:
 	case exec.SysTimerSetTime:
 	case exec.SysTimerDelete:
-	case exec.SysClockGetTime:
 	default:
 		m.Traceback()
 		panic(fmt.Sprintf("unrecognized syscall: %d", syscallNum))

cannon/mipsevm/tests/evm_common_test.go

@@ -485,8 +485,8 @@ func TestHelloEVM(t *testing.T) {
 				// verify the post-state matches.
 				// TODO: maybe more readable to decode the evmPost state, and do attribute-wise comparison.
 				goPost, _ := goVm.GetState().EncodeWitness()
-				require.Equal(t, hexutil.Bytes(goPost).String(), hexutil.Bytes(evmPost).String(),
-					"mipsevm produced different state than EVM")
+				require.Equalf(t, hexutil.Bytes(goPost).String(), hexutil.Bytes(evmPost).String(),
+					"mipsevm produced different state than EVM. insn: %x", insn)
 			}
 			end := time.Now()
 			delta := end.Sub(start)

cannon/mipsevm/testutil/oracle.go

@@ -120,14 +120,19 @@ func ClaimTestOracle(t *testing.T) (po mipsevm.PreimageOracle, stdOut string, st
 	return oracle, fmt.Sprintf("computing %d * %d + %d\nclaim %d is good!\n", s, a, b, s*a+b), "started!"
 }
 
-func AllocOracle(t *testing.T, numAllocs int) *TestOracle {
+func AllocOracle(t *testing.T, numAllocs int, allocSize int) *TestOracle {
 	return &TestOracle{
 		hint: func(v []byte) {},
 		getPreimage: func(k [32]byte) []byte {
-			if k != preimage.LocalIndexKey(0).PreimageKey() {
+			switch k {
+			case preimage.LocalIndexKey(0).PreimageKey():
+				return binary.LittleEndian.AppendUint64(nil, uint64(numAllocs))
+			case preimage.LocalIndexKey(1).PreimageKey():
+				return binary.LittleEndian.AppendUint64(nil, uint64(allocSize))
+			default:
 				t.Fatalf("invalid preimage request for %x", k)
 			}
-			return binary.LittleEndian.AppendUint64(nil, uint64(numAllocs))
+			panic("unreachable")
 		},
 	}
 }

cannon/testdata/example/alloc/main.go

@@ -12,20 +12,27 @@ func main() {
 	var mem []byte
 	po := preimage.NewOracleClient(preimage.ClientPreimageChannel())
 	numAllocs := binary.LittleEndian.Uint64(po.Get(preimage.LocalIndexKey(0)))
+	allocSize := binary.LittleEndian.Uint64(po.Get(preimage.LocalIndexKey(1)))
 
-	fmt.Printf("alloc program. numAllocs=%d\n", numAllocs)
+	fmt.Printf("alloc program. numAllocs=%d allocSize=%d\n", numAllocs, allocSize)
 	var alloc int
 	for i := 0; i < int(numAllocs); i++ {
-		mem = make([]byte, 32*1024*1024)
+		mem = make([]byte, allocSize)
 		alloc += len(mem)
 		// touch a couple pages to prevent the runtime from overcommitting memory
 		for j := 0; j < len(mem); j += 1024 {
 			mem[j] = 0xFF
 		}
-		fmt.Printf("allocated %d bytes\n", alloc)
+		printGCStats(alloc)
 	}
 
+	fmt.Println("alloc program exit")
+	printGCStats(alloc)
+}
+
+func printGCStats(alloc int) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
-	fmt.Printf("alloc program exit. memstats: heap_alloc=%d frees=%d mallocs=%d\n", m.HeapAlloc, m.Frees, m.Mallocs)
+	fmt.Printf("allocated %d bytes. memstats: heap_alloc=%d next_gc=%d frees=%d mallocs=%d num_gc=%d\n",
+		alloc, m.HeapAlloc, m.NextGC, m.Frees, m.Mallocs, m.NumGC)
 }

op-chain-ops/srcmap/solutil.go

@@ -235,8 +235,15 @@ func (s *SourceMapTracer) info(codeAddr common.Address, pc uint64) string {
 func (s *SourceMapTracer) OnOpCode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) {
 	op := vm.OpCode(opcode)
 	if op.IsPush() {
-		start := uint64(op) - uint64(vm.PUSH1) + 1
-		val := scope.StackData()[:start]
+		var val []byte
+		sc, ok := scope.(*vm.ScopeContext)
+		if ok {
+			start := pc + 1
+			end := uint64(op) - uint64(vm.PUSH1) + 1
+			val = sc.Contract.Code[start : start+end]
+		} else {
+			val = []byte("N/A")
+		}
 		fmt.Fprintf(s.out, "%-40s : pc %x opcode %s (%x)\n", s.info(scope.Address(), pc), pc, op.String(), val)
 		return
 	}

packages/contracts-bedrock/scripts/go-ffi/differential-testing.go

@@ -359,10 +359,10 @@ func DiffTestUtils() {
 		// Print the output
 		fmt.Print(hexutil.Encode(packed[32:]))
 	case "cannonMemoryProof":
-		// <pc, insn, [memAddr, memValue]>
+		// <pc, insn, [memAddr, memValue], [memAddr, memValue]>
 		mem := memory.NewMemory()
-		if len(args) != 3 && len(args) != 5 {
-			panic("Error: cannonMemoryProofWithProof requires 2 or 4 arguments")
+		if len(args) != 3 && len(args) != 5 && len(args) != 7 {
+			panic("Error: cannonMemoryProofWithProof requires 2, 4, or 6 arguments")
 		}
 		pc, err := strconv.ParseUint(args[1], 10, 32)
 		checkErr(err, "Error decoding addr")
@@ -371,14 +371,22 @@ func DiffTestUtils() {
 		mem.SetMemory(uint32(pc), uint32(insn))
 
 		var insnProof, memProof [896]byte
-		if len(args) == 5 {
+		if len(args) >= 5 {
 			memAddr, err := strconv.ParseUint(args[3], 10, 32)
 			checkErr(err, "Error decoding memAddr")
 			memValue, err := strconv.ParseUint(args[4], 10, 32)
 			checkErr(err, "Error decoding memValue")
 			mem.SetMemory(uint32(memAddr), uint32(memValue))
 			memProof = mem.MerkleProof(uint32(memAddr))
 		}
+		if len(args) == 7 {
+			memAddr, err := strconv.ParseUint(args[5], 10, 32)
+			checkErr(err, "Error decoding memAddr")
+			memValue, err := strconv.ParseUint(args[6], 10, 32)
+			checkErr(err, "Error decoding memValue")
+			mem.SetMemory(uint32(memAddr), uint32(memValue))
+			memProof = mem.MerkleProof(uint32(memAddr))
+		}
 		insnProof = mem.MerkleProof(uint32(pc))
 
 		output := struct {
@@ -391,6 +399,39 @@ func DiffTestUtils() {
 		packed, err := cannonMemoryProofArgs.Pack(&output)
 		checkErr(err, "Error encoding output")
 		fmt.Print(hexutil.Encode(packed[32:]))
+	case "cannonMemoryProof2":
+		// <pc, insn, [memAddr, memValue], memAddr2>
+		mem := memory.NewMemory()
+		if len(args) != 6 {
+			panic("Error: cannonMemoryProofWithProof2 requires 5 arguments")
+		}
+		pc, err := strconv.ParseUint(args[1], 10, 32)
+		checkErr(err, "Error decoding addr")
+		insn, err := strconv.ParseUint(args[2], 10, 32)
+		checkErr(err, "Error decoding insn")
+		mem.SetMemory(uint32(pc), uint32(insn))
+
+		var memProof [896]byte
+		memAddr, err := strconv.ParseUint(args[3], 10, 32)
+		checkErr(err, "Error decoding memAddr")
+		memValue, err := strconv.ParseUint(args[4], 10, 32)
+		checkErr(err, "Error decoding memValue")
+		mem.SetMemory(uint32(memAddr), uint32(memValue))
+
+		memAddr2, err := strconv.ParseUint(args[5], 10, 32)
+		checkErr(err, "Error decoding memAddr")
+		memProof = mem.MerkleProof(uint32(memAddr2))
+
+		output := struct {
+			MemRoot common.Hash
+			Proof   []byte
+		}{
+			MemRoot: mem.MerkleRoot(),
+			Proof:   memProof[:],
+		}
+		packed, err := cannonMemoryProofArgs.Pack(&output)
+		checkErr(err, "Error encoding output")
+		fmt.Print(hexutil.Encode(packed[32:]))
 	case "cannonMemoryProofWrongLeaf":
 		// <pc, insn, memAddr, memValue>
 		mem := memory.NewMemory()

packages/contracts-bedrock/semver-lock.json

@@ -148,8 +148,8 @@
     "sourceCodeHash": "0xb6e219e8c2d81d75c48a1459907609e9096fe032a7447c88cd3e0d134752ac8e"
   },
   "src/cannon/MIPS2.sol": {
-    "initCodeHash": "0x36b7c32cf9eba05e6db44910a25c800b801c075f8e053eca9515c6e0e4d8a902",
-    "sourceCodeHash": "0xa307c44a2d67bc84e75f4b7341345ed236da2e63c1f3f442416f14cd262126bf"
+    "initCodeHash": "0x10b76556ecb33bdbb152d3634a9a52ca65df3df9d1dc2412fcba36c0bc5110b7",
+    "sourceCodeHash": "0x74a9afea105228f5569268cccb5ab713a24cf3547cfed0589da29c11f67d10c2"
   },
   "src/cannon/PreimageOracle.sol": {
     "initCodeHash": "0xce7a1c3265e457a05d17b6d1a2ef93c4639caac3733c9cf88bfd192eae2c5788",