diff --git a/packages/testing/src/execution_testing/benchmark/benchmark_code_generator.py b/packages/testing/src/execution_testing/benchmark/benchmark_code_generator.py index dd8deb42f4..8a77c4eb96 100644 --- a/packages/testing/src/execution_testing/benchmark/benchmark_code_generator.py +++ b/packages/testing/src/execution_testing/benchmark/benchmark_code_generator.py @@ -3,9 +3,9 @@ optimized bytecode patterns. """ -from dataclasses import dataclass +from dataclasses import dataclass, field -from execution_testing.base_types import Address +from execution_testing.base_types import Address, Storage from execution_testing.forks import Fork from execution_testing.specs.benchmark import BenchmarkCodeGenerator from execution_testing.test_types import Alloc @@ -17,6 +17,7 @@ class JumpLoopGenerator(BenchmarkCodeGenerator): """Generates bytecode that loops execution using JUMP operations.""" contract_balance: int = 0 + contract_storage: Storage = field(default_factory=Storage) def deploy_contracts(self, *, pre: Alloc, fork: Fork) -> Address: """Deploy the looping contract.""" @@ -31,7 +32,9 @@ def deploy_contracts(self, *, pre: Alloc, fork: Fork) -> Address: fork=fork, ) self._contract_address = pre.deploy_contract( - code=code, balance=self.contract_balance + code=code, + balance=self.contract_balance, + storage=self.contract_storage, ) return self._contract_address diff --git a/tests/benchmark/stateful/bloatnet/test_single_opcode.py b/tests/benchmark/stateful/bloatnet/test_single_opcode.py index 664ba48db0..51d53705e6 100644 --- a/tests/benchmark/stateful/bloatnet/test_single_opcode.py +++ b/tests/benchmark/stateful/bloatnet/test_single_opcode.py @@ -10,16 +10,23 @@ import json import math from pathlib import Path +from typing import Tuple import pytest from execution_testing import ( + AccessList, Account, + Address, Alloc, + BenchmarkTestFiller, Block, BlockchainTestFiller, Bytecode, Fork, + Hash, + JumpLoopGenerator, Op, + Storage, Transaction, While, ) @@ -394,3 +401,492 @@ def test_sstore_erc20_approve( blocks=[Block(txs=txs)], post=post, ) + + +def sstore_helper_contract( + *, + sloads_before_sstore: bool, + key_warm: bool, + original_value: int, + new_value: int, +) -> Tuple[Bytecode, Bytecode, Bytecode]: + """ + Storage contract for benchmark slot access. + + # Calldata Layout: + # - CALLDATA[0..31]: Starting slot + # - CALLDATA[32..63]: Ending slot + # - CALLDATA[64..95]: Value to write + + Returns: + - setup: Bytecode of the setup of the contract + - loop: Bytecode of the loop of the contract + - cleanup: Bytecode of the cleanup of the contract + + """ + setup = Bytecode() + loop = Bytecode() + cleanup = Bytecode() + + setup += ( + Op.CALLDATALOAD(32) # end_slot + + Op.CALLDATALOAD(64) # value + + Op.CALLDATALOAD(0) # start_slot = counter + ) + # [counter, value, end_slot] + + loop += Op.JUMPDEST + # Loop Body: Store Value at Start Slot + Counter + if sloads_before_sstore: + loop += Op.DUP1 # [counter, counter, value, end_slot] + loop += Op.SLOAD(key_warm=key_warm) + loop += Op.POP + loop += Op.DUP2 # [value, counter, value, end_slot] + loop += Op.DUP2 # [counter, value, counter, value, end_slot] + loop += Op.SSTORE( + key_warm=True, + original_value=original_value, + new_value=new_value, + ) + else: + loop += Op.DUP2 # [value, counter, value, end_slot] + loop += Op.DUP2 # [counter, value, counter, value, end_slot] + loop += Op.SSTORE( # STORAGE[counter, value] = value + key_warm=key_warm, + original_value=original_value, + new_value=new_value, + ) + + # Loop Post: Increment Counter + loop += Op.PUSH1(1) + loop += Op.ADD + # [counter + 1, value, end_slot] + + # Loop Condition: Counter < Num Slots + loop += Op.DUP3 # [end_slot, counter + 1, value, end_slot] + loop += Op.DUP2 # [counter + 1, end_slot, counter + 1, value, end_slot] + loop += Op.LT # [counter + 1 < end_slot, counter + 1, value, end_slot] + loop += Op.ISZERO + loop += Op.ISZERO + loop += Op.PUSH1(len(setup)) + loop += Op.JUMPI + # [counter + 1, value, end_slot] + + # Cleanup: Stop + cleanup += Op.STOP + + return setup, loop, cleanup + + +@pytest.mark.parametrize("use_access_list", [True, False]) +@pytest.mark.parametrize("sloads_before_sstore", [True, False]) +@pytest.mark.parametrize("num_contracts", [1, 5, 10]) +@pytest.mark.parametrize( + "initial_value,write_value", + [ + pytest.param(0, 0, id="zero_to_zero"), + pytest.param(0, 0xDEADBEEF, id="zero_to_nonzero"), + pytest.param(0xDEADBEEF, 0, id="nonzero_to_zero"), + pytest.param(0xDEADBEEF, 0xBEEFBEEF, id="nonzero_to_diff"), + pytest.param(0xDEADBEEF, 0xDEADBEEF, id="nonzero_to_same"), + ], +) +def test_sstore_variants( + benchmark_test: BenchmarkTestFiller, + fork: Fork, + pre: Alloc, + tx_gas_limit: int, + gas_benchmark_value: int, + use_access_list: bool, + sloads_before_sstore: bool, + num_contracts: int, + initial_value: int, + write_value: int, +) -> None: + """ + Benchmark SSTORE instruction with various configurations. + + Variants: + - use_access_list: Warm storage slots via access list + - sloads_before_sstore: Number of SLOADs per slot before SSTORE + - num_contracts: Number of contract instances (cold storage writes) + - initial_value/write_value: Storage transitions + (zero_to_zero, zero_to_nonzero, nonzero_to_zero, nonzero_to_nonzero) + """ + ( + contract_setup, + contract_loop, + contract_cleanup, + ) = sstore_helper_contract( + sloads_before_sstore=sloads_before_sstore, + key_warm=use_access_list, + original_value=initial_value, + new_value=write_value, + ) + contract = contract_setup + contract_loop + contract_cleanup + + gas_per_contract = gas_benchmark_value // num_contracts + gas_limit_cap = fork.transaction_gas_limit_cap() + intrinsic_gas_cost_calc = fork.transaction_intrinsic_cost_calculator() + + def get_calldata(iteration_count: int, start_slot: int) -> bytes: + return ( + Hash(start_slot) + + Hash(start_slot + iteration_count) + + Hash(write_value) + ) + + def get_access_list( + iteration_count: int, start_slot: int, contract_addr: Address + ) -> list[AccessList] | None: + if use_access_list: + storage_keys = [ + Hash(i) + for i in range(start_slot, start_slot + iteration_count) + ] + return [ + AccessList( + address=contract_addr, + storage_keys=storage_keys, + ) + ] + return None + + def calc_gas_consumed( + iteration_count: int, start_slot: int, contract_addr: Address + ) -> int: + intrinsic_gas_cost = intrinsic_gas_cost_calc( + calldata=get_calldata(iteration_count, start_slot), + access_list=get_access_list( + iteration_count, start_slot, contract_addr + ), + return_cost_deducted_prior_execution=True, + ) + overhead_gas = ( + contract_setup.gas_cost(fork) + + contract_cleanup.gas_cost(fork) + + intrinsic_gas_cost + ) + iteration_cost = contract_loop.gas_cost(fork) * iteration_count + return overhead_gas + iteration_cost + + def calc_gas_required( + iteration_count: int, start_slot: int, contract_addr: Address + ) -> int: + gsc = fork.gas_costs() + # SSTORE requires a minimum gas of G_CALL_STIPEND to operate. + # TODO: Correct fix is to introduce bytecode.gas_required. + return ( + calc_gas_consumed(iteration_count, start_slot, contract_addr) + + gsc.G_CALL_STIPEND + ) + + # Calculate how many slots per contract per transaction are required + iteration_counts: list[int] = [] + remaining_gas = gas_per_contract + start_slot = 0 + while remaining_gas > 0: + gas_limit = ( + min(remaining_gas, gas_limit_cap) + if gas_limit_cap is not None + else remaining_gas + ) + if calc_gas_required(0, start_slot, Address(0)) > gas_limit: + break + + # Binary search the optimal number of iterations given the gas limit + low, high = 1, 2 + while calc_gas_required(high, start_slot, Address(0)) <= gas_limit: + high *= 2 + + while low < high: + mid = (low + high) // 2 + if calc_gas_required(mid, start_slot, Address(0)) > gas_limit: + high = mid + else: + low = mid + 1 + + iteration_count = low - 1 + iteration_counts.append(iteration_count) + start_slot += iteration_count + remaining_gas -= calc_gas_required( + iteration_count, start_slot, Address(0) + ) + + assert len(iteration_counts) > 0, ( + f"No iteration counts found for {num_contracts} contracts" + ) + + slots_per_contract = sum(iteration_counts) + + txs: list[Transaction] = [] + post = {} + + gas_used = 0 + for _ in range(num_contracts): + initial_storage = Storage() + + if initial_value != 0: + for i in range(slots_per_contract): + initial_storage[i] = initial_value + + contract_addr = pre.deploy_contract( + code=contract, + storage=initial_storage, + ) + + start_slot = 0 + for iteration_count in iteration_counts: + calldata = get_calldata(iteration_count, start_slot) + access_list = get_access_list( + iteration_count, start_slot, contract_addr + ) + tx_gas_limit = calc_gas_required( + iteration_count, start_slot, contract_addr + ) + tx_gas_consumed = calc_gas_consumed( + iteration_count, start_slot, contract_addr + ) + max_refund = tx_gas_consumed // 5 + refund = min( + contract_loop.refund(fork) * iteration_count, max_refund + ) + gas_used += tx_gas_consumed - refund + + tx = Transaction( + to=contract_addr, + data=calldata, + gas_limit=tx_gas_limit, + sender=pre.fund_eoa(), + access_list=access_list, + ) + txs.append(tx) + + start_slot += iteration_count + + expected_storage = Storage() + for i in range(slots_per_contract): + expected_storage[i] = write_value + + post[contract_addr] = Account( + code=contract, + storage=expected_storage, + ) + + benchmark_test( + blocks=[Block(txs=txs)], + post=post, + expected_benchmark_gas_used=gas_used, + ) + + +def sload_helper_contract( + *, key_warm: bool +) -> Tuple[Bytecode, Bytecode, Bytecode]: + """ + Storage contract for benchmark slot access. + + # Calldata Layout: + # - CALLDATA[0..31]: Starting slot + # - CALLDATA[32..63]: Ending slot + """ + setup = Bytecode() + loop = Bytecode() + cleanup = Bytecode() + + setup += Op.CALLDATALOAD(32) # end_slot + setup += Op.CALLDATALOAD(0) # start slot = counter + # [counter, end_slot] + + loop += Op.JUMPDEST + + # Loop Body: Load key from storage + loop += Op.DUP1 + loop += Op.SLOAD(key_warm=key_warm) + loop += Op.POP + # [counter, end_slot] + + # Loop Post: Increment Counter + loop += Op.PUSH1(1) + loop += Op.ADD + # [counter + 1, end_slot] + + # Loop Condition: Counter < Num Slots + loop += Op.DUP2 # [end_slot, counter + 1, end_slot] + loop += Op.DUP2 # [counter + 1, end_slot, counter + 1, end_slot] + loop += Op.LT # [counter + 1 < end_slot, counter + 1, end_slot] + loop += Op.ISZERO + loop += Op.ISZERO + loop += Op.PUSH1(len(setup)) + loop += Op.JUMPI + # [counter + 1, value, end_slot] + + # Cleanup: Stop + cleanup += Op.STOP + + return setup, loop, cleanup + + +@pytest.mark.parametrize("warm_slots", [False, True]) +@pytest.mark.parametrize("storage_keys_pre_set", [False, True]) +def test_storage_sload_benchmark( + benchmark_test: BenchmarkTestFiller, + pre: Alloc, + fork: Fork, + gas_benchmark_value: int, + warm_slots: bool, + storage_keys_pre_set: bool, + tx_gas_limit: int, +) -> None: + """ + Benchmark SLOAD instruction with various configurations. + + Variants: + - warm_slots: Warm storage slots via access list + - storage_keys_pre_set: Whether the storage keys are pre-set + """ + contract_setup, contract_loop, contract_cleanup = sload_helper_contract( + key_warm=warm_slots + ) + contract = contract_setup + contract_loop + contract_cleanup + + gas_limit_cap = fork.transaction_gas_limit_cap() + intrinsic_gas_cost_calc = fork.transaction_intrinsic_cost_calculator() + + def get_calldata(iteration_count: int, start_slot: int) -> bytes: + return Hash(start_slot) + Hash(start_slot + iteration_count) + + def get_access_list( + iteration_count: int, start_slot: int, contract_addr: Address + ) -> list[AccessList] | None: + if warm_slots: + storage_keys = [ + Hash(i) + for i in range(start_slot, start_slot + iteration_count) + ] + return [ + AccessList( + address=contract_addr, + storage_keys=storage_keys, + ) + ] + return None + + def calc_gas_required( + iteration_count: int, start_slot: int, contract_addr: Address + ) -> int: + intrinsic_gas_cost = intrinsic_gas_cost_calc( + calldata=get_calldata(iteration_count, start_slot), + access_list=get_access_list( + iteration_count, start_slot, contract_addr + ), + return_cost_deducted_prior_execution=True, + ) + overhead_gas = ( + contract_setup.gas_cost(fork) + + contract_cleanup.gas_cost(fork) + + intrinsic_gas_cost + ) + iteration_cost = contract_loop.gas_cost(fork) * iteration_count + return overhead_gas + iteration_cost + + # Calculate how many slots per transaction are required + iteration_counts: list[int] = [] + remaining_gas = gas_benchmark_value + start_slot = 0 + while remaining_gas > 0: + gas_limit = ( + min(remaining_gas, gas_limit_cap) + if gas_limit_cap is not None + else remaining_gas + ) + if calc_gas_required(0, start_slot, Address(0)) > gas_limit: + break + + # Binary search the optimal number of iterations given the gas limit + low, high = 1, 2 + while calc_gas_required(high, start_slot, Address(0)) <= gas_limit: + high *= 2 + + while low < high: + mid = (low + high) // 2 + if calc_gas_required(mid, start_slot, Address(0)) > gas_limit: + high = mid + else: + low = mid + 1 + + iteration_count = low - 1 + iteration_counts.append(iteration_count) + start_slot += iteration_count + remaining_gas -= calc_gas_required( + iteration_count, start_slot, Address(0) + ) + + assert len(iteration_counts) > 0, "No iteration counts found" + + slot_count = sum(iteration_counts) + + initial_storage = Storage() + if storage_keys_pre_set: + for i in range(slot_count): + initial_storage[i] = 1 + + contract_addr = pre.deploy_contract( + code=contract, + storage=initial_storage, + ) + + start_slot = 0 + txs: list[Transaction] = [] + gas_used = 0 + for iteration_count in iteration_counts: + calldata = get_calldata(iteration_count, start_slot) + access_list = get_access_list( + iteration_count, start_slot, contract_addr + ) + tx_gas_limit = calc_gas_required( + iteration_count, start_slot, contract_addr + ) + gas_used += tx_gas_limit + + tx = Transaction( + to=contract_addr, + data=calldata, + gas_limit=tx_gas_limit, + sender=pre.fund_eoa(), + access_list=access_list, + ) + txs.append(tx) + + start_slot += iteration_count + + benchmark_test( + pre=pre, + blocks=[Block(txs=txs)], + expected_benchmark_gas_used=gas_used, + ) + + +@pytest.mark.parametrize("storage_keys_pre_set", [False, True]) +def test_storage_sload_same_key_benchmark( + benchmark_test: BenchmarkTestFiller, + storage_keys_pre_set: bool, +) -> None: + """ + Benchmark SLOAD instruction when loading the same key over and over. + + Variants: + - storage_keys_pre_set: The key is pre-set to a non-zero value. + """ + contract_storage = Storage() + if storage_keys_pre_set: + contract_storage[1] = 1 + + benchmark_test( + target_opcode=Op.SLOAD, + code_generator=JumpLoopGenerator( + setup=Op.PUSH1(1) if storage_keys_pre_set else Op.PUSH0, + attack_block=Op.SLOAD, + contract_storage=contract_storage, + ), + )