Skip to content

perf(l1): lazy BAL cursor for per-tx parallel execution #6669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
edg-l merged 8 commits into from
May 22, 2026
Merged

perf(l1): lazy BAL cursor for per-tx parallel execution #6669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
a3c1baa
perf(l1): lazy BAL cursor for per-tx parallel execution
edg-l May 18, 2026
5b2815b
docs(changelog): add lazy BAL cursor perf entry
edg-l May 19, 2026
2be9ef7
fix(l1): lazy BAL cursor must precede shared_base in load_account
edg-l May 19, 2026
f7caa96
perf(l1): O(1) BAL slot lookup in eager+lazy seeding
edg-l May 20, 2026
df05e95
fix(l1): cfg-gate SlotChange import in gen_db
edg-l May 20, 2026
7efae28
style(l1): cargo fmt SlotChange import ordering
edg-l May 20, 2026
8c5a220
docs(levm): note storage-lazy invariant on BAL has_all_info shortcut
edg-l May 20, 2026
df9a356
refactor(l1): address iovoid review on bal-lazy-cursor
edg-l May 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## Perf

### 2026-05-19

- Lazy BAL cursor for per-tx parallel execution [#6669](https://github.com/lambdaclass/ethrex/pull/6669)

### 2026-05-15

- Replace synchronous disk I/O with async operations in snap sync [#6113](https://github.com/lambdaclass/ethrex/pull/6113)
Expand Down
20 changes: 19 additions & 1 deletion crates/common/types/block_access_list.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BTreeSet};

use crate::constants::{EMPTY_BLOCK_ACCESS_LIST_HASH, SYSTEM_ADDRESS};
use crate::utils::keccak;
use crate::utils::{keccak, u256_to_h256};

/// Encode a slice of items in sorted order without cloning.
fn encode_sorted_by<T, K, F>(items: &[T], buf: &mut dyn BufMut, key_fn: F)
Expand Down Expand Up @@ -560,6 +560,8 @@ impl BlockAccessList {
FxHashMap::with_capacity_and_hasher(self.inner.len(), Default::default());
let mut tx_to_accounts: FxHashMap<u32, Vec<usize>> = FxHashMap::default();
let mut accounts_by_min_index: Vec<(u32, usize)> = Vec::new();
let mut slot_idx_by_account: Vec<FxHashMap<H256, usize>> =
Vec::with_capacity(self.inner.len());

for (i, acct) in self.inner.iter().enumerate() {
addr_to_idx.insert(acct.address, i);
Expand Down Expand Up @@ -588,6 +590,15 @@ impl BlockAccessList {
for idx in seen_indices {
tx_to_accounts.entry(idx).or_default().push(i);
}

// Per-account slot → storage_changes index map for O(1) lookup on
// lazy-cursor cache miss. Empty for accounts with no storage writes.
let mut slot_map: FxHashMap<H256, usize> =
FxHashMap::with_capacity_and_hasher(acct.storage_changes.len(), Default::default());
for (sc_idx, sc) in acct.storage_changes.iter().enumerate() {
slot_map.insert(u256_to_h256(sc.slot), sc_idx);
}
slot_idx_by_account.push(slot_map);
}

accounts_by_min_index.sort_unstable_by_key(|(min_idx, _)| *min_idx);
Expand All @@ -596,12 +607,14 @@ impl BlockAccessList {
addr_to_idx,
tx_to_accounts,
accounts_by_min_index,
slot_idx_by_account,
}
}
}

/// Pre-computed index for fast per-tx BAL validation lookups.
/// Built once per block, shared read-only across parallel tx validations.
#[derive(Clone)]
pub struct BalAddressIndex {
/// Maps each address in the BAL to its index in `BlockAccessList.inner`.
pub addr_to_idx: FxHashMap<Address, usize>,
Expand All @@ -611,6 +624,11 @@ pub struct BalAddressIndex {
/// Used by `seed_db_from_bal` to skip accounts with no changes at indices <= max_idx.
/// Only includes accounts that have at least one mutation (balance/nonce/code/storage write).
pub accounts_by_min_index: Vec<(u32, usize)>,
/// Per-account slot → `storage_changes` index map. Lets `seed_one_storage_slot_from_bal`
/// resolve a slot key to its `SlotChange` in O(1) instead of a linear scan. Indexed by
/// the same `acct_idx` used by `addr_to_idx`; empty inner map for accounts with no
/// storage writes. Slot uniqueness is enforced by canonical-ordering validation.
pub slot_idx_by_account: Vec<FxHashMap<H256, usize>>,
}

/// Binary search for exact match at `idx` in balance changes (sorted by block_access_index).
Expand Down
163 changes: 38 additions & 125 deletions crates/vm/backends/levm/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ use ethrex_levm::constants::{
};
use ethrex_levm::db::gen_db::GeneralizedDatabase;
#[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
use ethrex_levm::db::gen_db::{
LazyBalCursor, code_from_bal, post_value_at_or_before, seed_one_address_info_from_bal,
};
#[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
use ethrex_levm::db::{Database, gen_db::CacheDB};
use ethrex_levm::errors::{InternalError, TxValidationError};
#[cfg(feature = "perf_opcode_timings")]
Expand Down Expand Up @@ -455,6 +459,7 @@ impl LEVM {
// Withdrawal index is n_txs+1 in BAL; we use n_txs to avoid double-applying
// withdrawal balances (process_withdrawals handles those below).
let last_tx_idx = u32::try_from(block.body.transactions.len()).unwrap_or(u32::MAX);
// Eager seed retained: lazy_bal cursor is per-tx only; outer DB has no cursor.
Self::seed_db_from_bal(
db,
bal,
Expand Down Expand Up @@ -737,19 +742,6 @@ impl LEVM {
))
}

/// Convert BAL into `Vec<AccountUpdate>` for the merkleizer.
/// Compute code hash and optional `Code` object from raw bytecode in a BAL entry.
#[cfg(all(feature = "rayon", not(feature = "eip-8025")))]
fn code_from_bal(new_code: &Bytes) -> (H256, Option<Code>) {
if new_code.is_empty() {
(*EMPTY_KECCACK_HASH, None)
} else {
let code_obj = Code::from_bytecode(new_code.clone(), &ethrex_crypto::NativeCrypto);
let hash = code_obj.hash;
(hash, Some(code_obj))
}
}

///
/// For each account in the BAL, extracts the **final** post-block state
/// (highest `block_access_index` entry per field) and builds an AccountUpdate.
Expand Down Expand Up @@ -812,7 +804,7 @@ impl LEVM {

// Final code: last entry or prestate
let (code_hash, code) = if let Some(c) = acct_changes.code_changes.last() {
Self::code_from_bal(&c.new_code)
code_from_bal(&c.new_code)
} else {
(prestate.code_hash, None)
};
Expand Down Expand Up @@ -881,6 +873,12 @@ impl LEVM {
Ok(updates)
}

/// Eager BAL prefix seed — used only by the outer DB path (parallel-execution
/// fallback recovery and post-tx outer seed before request extraction).
/// Per-tx parallel execution uses `LazyBalCursor` in `execute_block_parallel`;
/// see also `seed_one_address_info_from_bal` and `seed_one_storage_slot_from_bal`
/// in `ethrex_levm::db::gen_db`.
///
/// Pre-seed a GeneralizedDatabase with BAL-derived state for a specific tx.
///
/// For each BAL-modified account, applies accumulated diffs with
Expand All @@ -898,118 +896,38 @@ impl LEVM {
max_idx: u32,
accounts_by_min_index: &[(u32, usize)],
) -> Result<(), EvmError> {
// Only visit accounts whose minimum change index <= max_idx.
let end = accounts_by_min_index.partition_point(|(min_idx, _)| *min_idx <= max_idx);
let bal_accounts = bal.accounts();
for &(_, acct_idx) in &accounts_by_min_index[..end] {
let acct_changes = &bal_accounts[acct_idx];
let addr = acct_changes.address;
seed_one_address_info_from_bal(db, bal, acct_idx, max_idx)
.map_err(|e| EvmError::Custom(format!("seed_db_from_bal: {e}")))?;

// Binary search (slices are sorted ascending by block_access_index):
// partition_point returns the number of elements <= max_idx.
let balance_pos = acct_changes
.balance_changes
.partition_point(|c| c.block_access_index <= max_idx);
let nonce_pos = acct_changes
.nonce_changes
.partition_point(|c| c.block_access_index <= max_idx);
let code_pos = acct_changes
.code_changes
.partition_point(|c| c.block_access_index <= max_idx);
// Each slot's slot_changes are sorted ascending by block_access_index,
// so if the first entry is <= max_idx, at least one change is in scope.
let acct_changes = &bal_accounts[acct_idx];
if acct_changes.storage_changes.is_empty() {
continue;
}
let any_storage = acct_changes.storage_changes.iter().any(|sc| {
sc.slot_changes
.first()
.is_some_and(|c| c.block_access_index <= max_idx)
});

if balance_pos == 0 && nonce_pos == 0 && !any_storage && code_pos == 0 {
if !any_storage {
continue;
}

// Compute code update before borrowing acc (borrow checker: can't access
// db.codes while acc holds a mutable borrow of db)
let code_update = if code_pos > 0 {
Some(Self::code_from_bal(
&acct_changes.code_changes[code_pos - 1].new_code,
))
} else {
None
};

// When BAL covers all account info fields (balance + nonce + code), insert
// a default LevmAccount directly to skip the store/shared_base lookup.
// For partial coverage, load from store to fill missing fields.
let has_all_info = balance_pos > 0 && nonce_pos > 0 && code_pos > 0;
if has_all_info {
use ethrex_common::types::AccountInfo;
let balance = acct_changes.balance_changes[balance_pos - 1].post_balance;
let nonce = acct_changes.nonce_changes[nonce_pos - 1].post_nonce;
let code_hash = code_update
.as_ref()
.map(|(h, _)| *h)
.unwrap_or(*EMPTY_KECCACK_HASH);
// NOTE: has_storage is false for newly inserted accounts. This is safe
// because this DB is only used for the parallel execution path (state
// comes from BAL, not get_state_transitions_tx). Do not reuse this DB
// for sequential fallback without fixing has_storage.
let acc = db
.current_accounts_state
.entry(addr)
.or_insert_with(|| LevmAccount {
info: AccountInfo::default(),
storage: FxHashMap::default(),
has_storage: false,
status: AccountStatus::Modified,
exists: true,
});
acc.info.balance = balance;
acc.info.nonce = nonce;
acc.info.code_hash = code_hash;
acc.mark_modified();
} else {
// Partial BAL coverage — load from store/shared_base, then overwrite
// the covered fields. get_account already caches, so get_account_mut
// will be a cache hit.
let addr = acct_changes.address;
if !db.current_accounts_state.contains_key(&addr) {
db.get_account(addr)
.map_err(|e| EvmError::Custom(format!("seed_db_from_bal load: {e}")))?;
let acc = db
.get_account_mut(addr)
.map_err(|e| EvmError::Custom(format!("seed bal: {e}")))?;

if balance_pos > 0 {
acc.info.balance = acct_changes.balance_changes[balance_pos - 1].post_balance;
}
if nonce_pos > 0 {
acc.info.nonce = acct_changes.nonce_changes[nonce_pos - 1].post_nonce;
}
if let Some((hash, _)) = &code_update {
acc.info.code_hash = *hash;
}
.map_err(|e| EvmError::Custom(format!("seed storage: {e}")))?;
}

// Apply storage changes (works for both paths since acc is now in current_accounts_state)
if any_storage {
let acc = db
.current_accounts_state
.get_mut(&addr)
.expect("account was just inserted");
for sc in &acct_changes.storage_changes {
let pos = sc
.slot_changes
.partition_point(|c| c.block_access_index <= max_idx);
if pos > 0 {
let key = ethrex_common::utils::u256_to_h256(sc.slot);
acc.storage.insert(key, sc.slot_changes[pos - 1].post_value);
}
let acc = db
.get_account_mut(addr)
.map_err(|e| EvmError::Custom(format!("seed storage mut: {e}")))?;
for sc in &acct_changes.storage_changes {
if let Some(value) = post_value_at_or_before(sc, max_idx) {
acc.storage
.insert(ethrex_common::utils::u256_to_h256(sc.slot), value);
}
}
Comment thread
edg-l marked this conversation as resolved.

// Insert code object after acc borrow is released
if let Some((hash, Some(code_obj))) = code_update {
db.codes.entry(hash).or_insert(code_obj);
}
}
Ok(())
}
Expand Down Expand Up @@ -1105,8 +1023,9 @@ impl LEVM {
.is_some_and(|a| a.storage.contains_key(key))
});

// Pre-compute capacity hint for per-tx DBs from BAL account count.
let bal_account_count = bal.accounts().len();
// Small capacity hint — per-tx DBs materialize only touched accounts via lazy_bal cursor.
let arc_bal = Arc::new(bal.clone());
let arc_idx = Arc::new(validation_index.clone());

// 2. Execute all txs in parallel (embarrassingly parallel, BAL-seeded).
// BAL validation is deferred to after the gas limit check (step 3) so that
Expand All @@ -1129,23 +1048,17 @@ impl LEVM {
let mut tx_db = GeneralizedDatabase::new_with_shared_base_and_capacity(
store.clone(),
system_seed.clone(),
bal_account_count,
32,
);
tx_db.lazy_bal = Some(LazyBalCursor {
bal: arc_bal.clone(),
bal_index: u32::try_from(tx_idx + 1).unwrap_or(u32::MAX),
index: arc_idx.clone(),
});
// Small capacity: parallel txs rarely nest >8 call frames, and
// over-allocating per-tx wastes memory across many rayon tasks.
let mut stack_pool = Vec::with_capacity(8);

// Pre-seed with BAL-derived intermediate state.
// BAL index: 0 = system calls, 1 = tx 0, 2 = tx 1, ...
// For tx at index i, we want state through BAL index i
// (= system calls + effects of txs 0..i-1).
Self::seed_db_from_bal(
&mut tx_db,
bal,
u32::try_from(tx_idx).unwrap_or(u32::MAX),
&validation_index.accounts_by_min_index,
)?;

// Enable accessed_accounts tracker (coarse) for `unaccessed_pure_accounts`
// diagnostics. Safe to over-report: used only to REMOVE entries from a
// extraneous-entry checklist.
Expand Down
Loading
Loading