From 42b30ea5aa7695184f7d6482c78f2467a5887b11 Mon Sep 17 00:00:00 2001 From: unnawut Date: Wed, 30 Jul 2025 18:07:46 +0700 Subject: [PATCH 01/47] feat: initial code for 3SF --- Cargo.lock | 3 + Cargo.toml | 1 + bin/ream/Cargo.toml | 1 + crates/common/consensus/lean/Cargo.toml | 2 + .../common/consensus/lean/src/attestation.rs | 16 - crates/common/consensus/lean/src/block.rs | 15 +- crates/common/consensus/lean/src/lib.rs | 207 ++++++++++- crates/common/consensus/lean/src/staker.rs | 345 ++++++++++++++++++ crates/common/consensus/lean/src/state.rs | 25 +- crates/common/consensus/lean/src/validator.rs | 9 - crates/common/consensus/lean/src/vote.rs | 32 ++ 11 files changed, 622 insertions(+), 34 deletions(-) delete mode 100644 crates/common/consensus/lean/src/attestation.rs create mode 100644 crates/common/consensus/lean/src/staker.rs delete mode 100644 crates/common/consensus/lean/src/validator.rs create mode 100644 crates/common/consensus/lean/src/vote.rs diff --git a/Cargo.lock b/Cargo.lock index a7a6f5ba6..6c93f6d63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5163,6 +5163,7 @@ dependencies = [ "ream-beacon-api-types", "ream-checkpoint-sync", "ream-consensus-beacon", + "ream-consensus-lean", "ream-consensus-misc", "ream-discv5", "ream-executor", @@ -5309,10 +5310,12 @@ name = "ream-consensus-lean" version = "0.1.0" dependencies = [ "alloy-primitives", + "ethereum_hashing", "ethereum_ssz", "ethereum_ssz_derive", "ream-pqc", "serde", + "serde_json", "ssz_types", "tree_hash", "tree_hash_derive", diff --git a/Cargo.toml b/Cargo.toml index f83c68d5a..d6ac4fcac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -119,6 +119,7 @@ ream-bls = { path = "crates/crypto/bls", features = ["zkcrypto"] } # Default fea ream-chain-beacon = { path = "crates/common/chain/beacon" } ream-checkpoint-sync = { path = "crates/common/checkpoint_sync" } ream-consensus-beacon = { path = "crates/common/consensus/beacon" } +ream-consensus-lean = { path = "crates/common/consensus/lean" } ream-consensus-misc = { path = "crates/common/consensus/misc" } ream-discv5 = { path = "crates/networking/discv5" } ream-execution-engine = { path = "crates/common/execution_engine" } diff --git a/bin/ream/Cargo.toml b/bin/ream/Cargo.toml index 3564d137c..08843dc13 100644 --- a/bin/ream/Cargo.toml +++ b/bin/ream/Cargo.toml @@ -33,6 +33,7 @@ ream-account-manager.workspace = true ream-beacon-api-types.workspace = true ream-checkpoint-sync.workspace = true ream-consensus-beacon.workspace = true +ream-consensus-lean.workspace = true ream-consensus-misc.workspace = true ream-discv5.workspace = true ream-executor.workspace = true diff --git a/crates/common/consensus/lean/Cargo.toml b/crates/common/consensus/lean/Cargo.toml index 0f46684e7..4ab7dc944 100644 --- a/crates/common/consensus/lean/Cargo.toml +++ b/crates/common/consensus/lean/Cargo.toml @@ -11,9 +11,11 @@ version.workspace = true [dependencies] alloy-primitives.workspace = true +ethereum_hashing.workspace = true ethereum_ssz.workspace = true ethereum_ssz_derive.workspace = true serde.workspace = true +serde_json.workspace = true ssz_types.workspace = true tree_hash.workspace = true tree_hash_derive.workspace = true diff --git a/crates/common/consensus/lean/src/attestation.rs b/crates/common/consensus/lean/src/attestation.rs deleted file mode 100644 index 23cefab95..000000000 --- a/crates/common/consensus/lean/src/attestation.rs +++ /dev/null @@ -1,16 +0,0 @@ -use ream_pqc::PQSignature; -use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use tree_hash_derive::TreeHash; - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct Attestation { - pub data: AttestationData, - pub signature: PQSignature, -} - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct AttestationData { - pub slot: u64, - pub index: u64, -} diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 0cf7be17d..98ab6f3b3 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -1,9 +1,11 @@ -use alloy_primitives::B256; +use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use tree_hash_derive::TreeHash; +use crate::Hash; + #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct SignedBlock { pub message: Block, @@ -16,7 +18,14 @@ pub struct SignedBlock { pub struct Block { pub slot: u64, pub proposer_index: u64, + pub parent: Option, + pub votes: Vec, + pub state_root: Option, +} - /// Empty root - pub body: B256, +impl Block { + pub fn compute_hash(&self) -> Hash { + let serialized = serde_json::to_string(self).unwrap(); + Hash::from_slice(&hash(serialized.as_bytes())) + } } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index a93e8576e..4da237967 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -1,4 +1,207 @@ -pub mod attestation; pub mod block; pub mod state; -pub mod validator; +pub mod staker; +pub mod vote; + +use alloy_primitives::B256; +use std::collections::HashMap; + +use crate::{ + block::Block, + state::State, + vote::Vote, +}; + +pub type Hash = B256; + +pub const ZERO_HASH: Hash = Hash::ZERO; + +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum QueueItem { + BlockItem(Block), + VoteItem(Vote), +} + +// We allow justification of slots either <= 5 or a perfect square or oblong after +// the latest finalized slot. This gives us a backoff technique and ensures +// finality keeps progressing even under high latency +pub fn is_justifiable_slot(finalized_slot: &usize, candidate_slot: &usize) -> bool { + assert!( + candidate_slot >= finalized_slot, + "Candidate slot ({candidate_slot}) is less than finalized slot ({finalized_slot})" + ); + + let delta = candidate_slot - finalized_slot; + + delta <= 5 + || (delta as f64).sqrt().fract() == 0.0 // any x^2 + || (delta as f64 + 0.25).sqrt() % 1.0 == 0.5 // any x^2+x +} + +// Given a state, output the new state after processing that block +pub fn process_block(pre_state: &State, block: &Block) -> State { + let mut state = pre_state.clone(); + + // Track historical blocks in the state + state.historical_block_hashes.push(block.parent); + state.justified_slots.push(false); + + while state.historical_block_hashes.len() < block.slot { + state.justified_slots.push(false); + state.historical_block_hashes.push(None); + } + + // Process votes + for vote in &block.votes { + // Ignore votes whose source is not already justified, + // or whose target is not in the history, or whose target is not a + // valid justifiable slot + if state.justified_slots[vote.source_slot] == false + || Some(vote.source) != state.historical_block_hashes[vote.source_slot] + || Some(vote.target) != state.historical_block_hashes[vote.target_slot] + || vote.target_slot <= vote.source_slot + || !is_justifiable_slot(&state.latest_finalized_slot, &vote.target_slot) + { + continue; + } + + // Track attempts to justify new hashes + if !state.justifications.contains_key(&vote.target) { + let mut empty_justifications = Vec::::with_capacity(state.config.num_validators); + empty_justifications.resize(state.config.num_validators, false); + + state + .justifications + .insert(vote.target, empty_justifications); + } + + if !state.justifications[&vote.target][vote.validator_id] { + state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id] = true; + } + + let count = state.justifications[&vote.target] + .iter() + .fold(0, |sum, justification| sum + *justification as usize); + + // If 2/3 voted for the same new valid hash to justify + if count == (2 * state.config.num_validators) / 3 { + state.latest_justified_hash = vote.target; + state.latest_justified_slot = vote.target_slot; + state.justified_slots[vote.target_slot] = true; + + state.justifications.remove(&vote.target).unwrap(); + + // Finalization: if the target is the next valid justifiable + // hash after the source + let mut is_target_next_valid_justifiable_slot = true; + + for slot in (vote.source_slot + 1)..vote.target_slot { + if is_justifiable_slot(&state.latest_finalized_slot, &slot) { + is_target_next_valid_justifiable_slot = false; + break; + } + } + + if is_target_next_valid_justifiable_slot { + state.latest_finalized_hash = vote.source; + state.latest_finalized_slot = vote.source_slot; + } + } + } + + state +} + +// Get the highest-slot justified block that we know about +pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { + let latest_justified_hash = post_states + .values() + .max_by_key(|state| state.latest_justified_slot) + .map(|state| state.latest_justified_hash); + + latest_justified_hash +} + +// Use LMD GHOST to get the head, given a particular root (usually the +// latest known justified block) +pub fn get_fork_choice_head( + blocks: &HashMap, + provided_root: &Hash, + votes: &Vec, + min_score: u64, +) -> Hash { + let mut root = *provided_root; + + // Start at genesis by default + if *root == ZERO_HASH { + root = blocks + .iter() + .min_by_key(|(_, block)| block.slot) + .map(|(hash, _)| *hash) + .unwrap(); + } + + // Sort votes by ascending slots to ensure that new votes are inserted last + let mut sorted_votes = votes.clone(); + sorted_votes.sort_by_key(|vote| vote.data.slot); + + // Prepare a map of validator_id -> their vote + let mut latest_votes = HashMap::::new(); + + for vote in votes { + latest_votes.insert(vote.data.validator_id, vote.clone()); + } + + // For each block, count the number of votes for that block. A vote + // for any descendant of a block also counts as a vote for that block + let mut vote_weights = HashMap::::new(); + + for (_validator_id, vote) in &latest_votes { + if blocks.contains_key(&vote.data.head) { + let mut block_hash = vote.data.head; + while blocks.get(&block_hash).unwrap().slot > blocks.get(&root).unwrap().slot { + let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); + vote_weights.insert(block_hash, current_weights + 1); + block_hash = blocks.get(&block_hash).unwrap().parent.unwrap(); + } + } + } + + // Identify the children of each block + let mut children_map = HashMap::>::new(); + + for (hash, block) in blocks { + if block.parent.is_some() && *vote_weights.get(hash).unwrap_or(&0) >= min_score { + match children_map.get_mut(&block.parent.unwrap()) { + Some(child_hashes) => { + child_hashes.push(*hash); + } + None => { + children_map.insert(block.parent.unwrap(), vec![*hash]); + } + } + } + } + + // Start at the root (latest justified hash or genesis) and repeatedly + // choose the child with the most latest votes, tiebreaking by slot then hash + let mut current_root = root; + + loop { + match children_map.get(¤t_root) { + None => { + break current_root; + } + Some(children) => { + current_root = *children + .iter() + .max_by_key(|child_hash| { + let vote_weight = vote_weights.get(*child_hash).unwrap_or(&0); + let slot = blocks.get(*child_hash).unwrap().slot; + (*vote_weight, slot, (*child_hash).clone()) + }) + .unwrap(); + } + } + } +} diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs new file mode 100644 index 000000000..d8a796f32 --- /dev/null +++ b/crates/common/consensus/lean/src/staker.rs @@ -0,0 +1,345 @@ +use ream_pqc::PublicKey; +use ream_pqc::PQSignature; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use std::{ + cell::RefCell, + collections::HashMap, + rc::{Rc, Weak}, +}; +use tree_hash_derive::TreeHash; + +use crate::{ + block::Block, + get_fork_choice_head, + get_latest_justified_hash, + Hash, + is_justifiable_slot, + process_block, + QueueItem, + state::State, + vote::{Vote, VoteData}, +}; + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +pub struct Staker { + pub validator_id: u64, + pub public_key: PublicKey, // Additional to 3SF-mini + pub network: Weak>, + pub chain: HashMap, + pub post_states: HashMap, + pub known_votes: Vec, + pub new_votes: Vec, + pub dependencies: HashMap>, + pub genesis_hash: Hash, + pub num_validators: u64, + pub safe_target: Option, + pub head: Hash, +} + +impl Staker { + pub fn new( + validator_id: u64, + network: &Rc>, + genesis_block: &Block, + genesis_state: &State, + ) -> Staker { + let genesis_hash = genesis_block.compute_hash(); + let mut chain = HashMap::::new(); + chain.insert(genesis_hash, genesis_block.clone()); + + let mut post_states = HashMap::::new(); + post_states.insert(genesis_hash, genesis_state.clone()); + + Staker { + validator_id, + public_key: PublicKey {}, + network: Rc::downgrade(network), + chain, + post_states, + known_votes: Vec::::new(), + new_votes: Vec::::new(), + dependencies: HashMap::>::new(), + genesis_hash, + num_validators: genesis_state.config.num_validators, + safe_target: None, + head: genesis_hash, + } + } + + /// A helper function that returns the Staker's network reference + /// by abstracting away Reference Counted implementation. + /// + /// Using `self.get_network()` is recommended over using `self.network` directly. + /// + /// Learn more: https://doc.rust-lang.org/std/rc/ + fn get_network(&self) -> Rc> { + self.network // Weak> + .upgrade() // Option>> + .unwrap() // Rc> + } + + pub fn latest_justified_hash(&self) -> Option { + get_latest_justified_hash(&self.post_states) + } + + pub fn latest_finalized_hash(&self) -> Option { + match self.post_states.get(&self.head) { + Some(state) => { + Some(state.latest_finalized_hash) + }, + None => None + } + } + + /// Compute the latest block that the staker is allowed to choose as the target + fn compute_safe_target(&self) -> Hash { + let justified_hash = get_latest_justified_hash(&self.post_states).unwrap(); + + get_fork_choice_head( + &self.chain, + &justified_hash, + &self.new_votes, + self.num_validators * 2 / 3, + ) + } + + /// Process new votes that the staker has received. Vote processing is done + /// at a particular time, because of safe target and view merge rule + fn accept_new_votes(&mut self) { + let mut known_votes = self.known_votes.clone().into_iter(); + + for new_vote in &self.new_votes { + if known_votes + .find(|known_vote| *known_vote == *new_vote) + .is_none() + { + self.known_votes.push(new_vote.clone()); + } + } + + self.new_votes.clear(); + self.recompute_head(); + } + + // Done upon processing new votes or a new block + fn recompute_head(&mut self) { + let justified_hash = get_latest_justified_hash(&self.post_states).unwrap(); + self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0); + } + + // Called every second + pub fn tick(&mut self) { + let time_in_slot = self.get_network().borrow().time % SLOT_DURATION; + + // t=0: propose a block + if time_in_slot == 0 { + if self.get_current_slot() % self.num_validators == self.validator_id { + // View merge mechanism: a node accepts attestations that it received + // <= 1/4 before slot start, or attestations in the latest block + self.accept_new_votes(); + self.propose_block(); + } + // t=1/4: vote + } else if time_in_slot == SLOT_DURATION / 4 { + self.vote(); + // t=2/4: compute the safe target (this must be done here to ensure + // that, assuming network latency assumptions are satisfied, anything that + // one honest node receives by this time, every honest node will receive by + // the general attestation deadline) + } else if time_in_slot == SLOT_DURATION * 2 / 4 { + self.safe_target = Some(self.compute_safe_target()); + // Deadline to accept attestations except for those included in a block + } else if time_in_slot == SLOT_DURATION * 3 / 4 { + self.accept_new_votes(); + } + } + + fn get_current_slot(&self) -> u64 { + self.get_network().borrow().time / SLOT_DURATION + 2 + } + + // Called when it's the staker's turn to propose a block + fn propose_block(&mut self) { + let new_slot = self.get_current_slot(); + + println!( + "proposing (Staker {}), head = {}", + self.validator_id, + self.chain.get(&self.head).unwrap().slot + ); + + let head_state = self.post_states.get(&self.head).unwrap(); + let mut new_block = Block { + slot: new_slot, + parent: Some(self.head), + votes: Vec::new(), + state_root: None, + }; + let mut state: State; + + // Keep attempt to add valid votes from the list of available votes + loop { + state = process_block(&head_state, &new_block); + + let mut new_votes_to_add = Vec::::new(); + for vote in self.known_votes.clone().into_iter() { + if vote.data.source == state.latest_justified_hash + && new_block.votes + .clone() + .into_iter() + .find(|v| *v == vote) + .is_none() + { + new_votes_to_add.push(vote); + } + } + + if new_votes_to_add.is_empty() { + break; + } + + new_block.votes.append(&mut new_votes_to_add); + } + + new_block.state_root = Some(state.compute_hash()); + let new_hash = new_block.compute_hash(); + + self.chain.insert(new_hash, new_block.clone()); + self.post_states.insert(new_hash, state); + + self.get_network() + .borrow_mut() + .submit(QueueItem::BlockItem(new_block), self.validator_id); + } + + // Called when it's the staker's turn to vote + fn vote(&mut self) { + let state = self.post_states.get(&self.head).unwrap(); + let mut target_block = self.chain.get(&self.head).unwrap(); + let safe_target = self.safe_target.unwrap_or(self.genesis_hash); + + // If there is no very recent safe target, then vote for the k'th ancestor + // of the head + for _ in 0..3 { + if target_block.slot > self.chain.get(&safe_target).unwrap().slot { + target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); + } + } + + // If the latest finalized slot is very far back, then only some slots are + // valid to justify, make sure the target is one of those + while !is_justifiable_slot(&state.latest_finalized_slot, &target_block.slot) { + target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); + } + + let vote_data = VoteData { + validator_id: self.validator_id, + slot: self.get_current_slot(), + head: self.head, + head_slot: self.chain.get(&self.head).unwrap().slot, + target: target_block.compute_hash(), + target_slot: target_block.slot, + source: state.latest_justified_hash, + source_slot: state.latest_justified_slot, + }; + + let vote = Vote { + data: vote_data, + signature: PQSignature {}, + }; + + println!( + "voting (Staker {}), head = {}, t = {}, s = {}", + self.validator_id, + &self.chain.get(&self.head).unwrap().slot, + &target_block.slot, + &state.latest_justified_slot + ); + + self.receive(&QueueItem::VoteItem(vote.clone())); + + self.get_network() + .borrow_mut() + .submit(QueueItem::VoteItem(vote), self.validator_id); + } + + // Called by the p2p network + fn receive(&mut self, queue_item: &QueueItem) { + match queue_item { + QueueItem::BlockItem(block) => { + let block_hash = block.compute_hash(); + + // If the block is already known, ignore it + if self.chain.contains_key(&block_hash) { + return; + } + + match self.post_states.get(&block.parent.unwrap()) { + Some(parent_state) => { + let state = process_block(&parent_state, &block); + + self.chain.insert(block_hash, block.clone()); + self.post_states.insert(block_hash, state); + + let mut known_votes = self.known_votes.clone().into_iter(); + + for vote in &block.votes { + if known_votes + .find(|known_vote| *known_vote == *vote) + .is_none() + { + self.known_votes.push(vote.clone()); + } + } + + self.recompute_head(); + + // Once we have received a block, also process all of + // its dependencies + if let Some(queue_items) = self.dependencies.get(&block_hash) { + for item in queue_items.clone() { + self.receive(&item); + } + + self.dependencies.remove(&block_hash); + } + } + None => { + // If we have not yet seen the block's parent, ignore for now, + // process later once we actually see the parent + self.dependencies + .entry(block.parent.unwrap()) + .or_insert_with(Vec::new) + .push(queue_item.clone()); + } + } + } + QueueItem::VoteItem(vote) => { + let is_known_vote = self + .known_votes + .clone() + .into_iter() + .find(|known_vote| known_vote == vote) + .is_some(); + let is_new_vote = self + .new_votes + .clone() + .into_iter() + .find(|new_vote| new_vote == vote) + .is_some(); + + if is_known_vote || is_new_vote { + return; + } else if self.chain.contains_key(&vote.data.head) { + self.new_votes.push(vote.clone()); + } else { + self.dependencies + .entry(vote.data.head) + .or_insert_with(Vec::new) + .push(queue_item.clone()); + } + } + } + } +} diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 448bce0e7..e8a28f013 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,14 +1,31 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::collections::HashMap; use tree_hash_derive::TreeHash; -use crate::validator::Validator; +use crate::{ + staker::Staker + Hash, +}; #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct BeamState { +pub struct LeanState { pub genesis_time: u64, + pub stakers: VariableList, - /// Up to 1 million validators - pub validators: VariableList, + pub latest_justified_hash: Hash, + pub latest_justified_slot: usize, + pub latest_finalized_hash: Hash, + pub latest_finalized_slot: usize, + pub historical_block_hashes: Vec>, + pub justified_slots: Vec, + pub justifications: HashMap>, +} + +impl State { + pub fn compute_hash(&self) -> Hash { + let serialized = serde_json::to_string(self).unwrap(); + Hash::from_slice(&hash(serialized.as_bytes())) + } } diff --git a/crates/common/consensus/lean/src/validator.rs b/crates/common/consensus/lean/src/validator.rs deleted file mode 100644 index c25d44414..000000000 --- a/crates/common/consensus/lean/src/validator.rs +++ /dev/null @@ -1,9 +0,0 @@ -use ream_pqc::PublicKey; -use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use tree_hash_derive::TreeHash; - -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct Validator { - pub public_key: PublicKey, -} diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs new file mode 100644 index 000000000..24232de80 --- /dev/null +++ b/crates/common/consensus/lean/src/vote.rs @@ -0,0 +1,32 @@ +use ream_pqc::PQSignature; +use ethereum_hashing::hash; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::TreeHash; + +use crate::Hash; + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +pub struct Vote { + pub data: VoteData, + pub signature: PQSignature, +} + +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +pub struct VoteData { + pub validator_id: u64, + pub slot: u64, + pub head: Hash, + pub head_slot: u64, + pub target: Hash, + pub target_slot: u64, + pub source: Hash, + pub source_slot: u64, +} + +impl Vote { + pub fn compute_hash(&self) -> Hash { + let serialized = serde_json::to_string(self).unwrap(); + Hash::from_slice(&hash(serialized.as_bytes())) + } +} From eeb0d8e7292f9d56f215265a62c76e5389fc2772 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 14:27:57 +0700 Subject: [PATCH 02/47] fix: quick fixes to get build passes first --- crates/common/consensus/lean/src/block.rs | 26 +++-- crates/common/consensus/lean/src/lib.rs | 73 ++++++++------ crates/common/consensus/lean/src/staker.rs | 110 ++++++++++----------- crates/common/consensus/lean/src/state.rs | 29 ++++-- crates/common/consensus/lean/src/vote.rs | 10 +- 5 files changed, 138 insertions(+), 110 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 98ab6f3b3..5c45d3b26 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -2,24 +2,32 @@ use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use tree_hash_derive::TreeHash; +use ssz_types::{ + VariableList, + typenum::{ + U16777216, // 2**24 + }, +}; -use crate::Hash; +use crate::{ + Hash, + vote::Vote, +}; -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +// TODO: Add back #[derive(TreeHash)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode)] pub struct SignedBlock { pub message: Block, pub signature: PQSignature, } -#[derive( - Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, Default, -)] +// TODO: Add back #[derive(TreeHash)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, Default)] pub struct Block { - pub slot: u64, - pub proposer_index: u64, + pub slot: usize, + pub proposer_index: usize, pub parent: Option, - pub votes: Vec, + pub votes: VariableList, pub state_root: Option, } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 4da237967..96696da04 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -4,19 +4,27 @@ pub mod staker; pub mod vote; use alloy_primitives::B256; +use serde::{Deserialize, Serialize}; +use ssz_types::{ + VariableList, + typenum::{ + U16777216, // 2**24 + }, +}; use std::collections::HashMap; use crate::{ block::Block, - state::State, + state::LeanState, vote::Vote, }; pub type Hash = B256; pub const ZERO_HASH: Hash = Hash::ZERO; +pub const SLOT_DURATION: usize = 12; -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum QueueItem { BlockItem(Block), VoteItem(Vote), @@ -39,16 +47,18 @@ pub fn is_justifiable_slot(finalized_slot: &usize, candidate_slot: &usize) -> bo } // Given a state, output the new state after processing that block -pub fn process_block(pre_state: &State, block: &Block) -> State { +pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { let mut state = pre_state.clone(); // Track historical blocks in the state - state.historical_block_hashes.push(block.parent); - state.justified_slots.push(false); + // TODO: proper error handlings + let _ = state.historical_block_hashes.push(block.parent); + let _ = state.justified_slots.push(false); while state.historical_block_hashes.len() < block.slot { - state.justified_slots.push(false); - state.historical_block_hashes.push(None); + // TODO: proper error handlings + let _ = state.justified_slots.push(false); + let _ = state.historical_block_hashes.push(None); } // Process votes @@ -56,46 +66,46 @@ pub fn process_block(pre_state: &State, block: &Block) -> State { // Ignore votes whose source is not already justified, // or whose target is not in the history, or whose target is not a // valid justifiable slot - if state.justified_slots[vote.source_slot] == false - || Some(vote.source) != state.historical_block_hashes[vote.source_slot] - || Some(vote.target) != state.historical_block_hashes[vote.target_slot] - || vote.target_slot <= vote.source_slot - || !is_justifiable_slot(&state.latest_finalized_slot, &vote.target_slot) + if state.justified_slots[vote.data.source_slot] == false + || Some(vote.data.source) != state.historical_block_hashes[vote.data.source_slot] + || Some(vote.data.target) != state.historical_block_hashes[vote.data.target_slot] + || vote.data.target_slot <= vote.data.source_slot + || !is_justifiable_slot(&state.latest_finalized_slot, &vote.data.target_slot) { continue; } // Track attempts to justify new hashes - if !state.justifications.contains_key(&vote.target) { - let mut empty_justifications = Vec::::with_capacity(state.config.num_validators); - empty_justifications.resize(state.config.num_validators, false); + if !state.justifications.contains_key(&vote.data.target) { + let mut empty_justifications = Vec::::with_capacity(state.num_validators); + empty_justifications.resize(state.num_validators, false); state .justifications - .insert(vote.target, empty_justifications); + .insert(vote.data.target, empty_justifications); } - if !state.justifications[&vote.target][vote.validator_id] { - state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id] = true; + if !state.justifications[&vote.data.target][vote.data.validator_id] { + state.justifications.get_mut(&vote.data.target).unwrap()[vote.data.validator_id] = true; } - let count = state.justifications[&vote.target] + let count = state.justifications[&vote.data.target] .iter() .fold(0, |sum, justification| sum + *justification as usize); // If 2/3 voted for the same new valid hash to justify - if count == (2 * state.config.num_validators) / 3 { - state.latest_justified_hash = vote.target; - state.latest_justified_slot = vote.target_slot; - state.justified_slots[vote.target_slot] = true; + if count == (2 * state.num_validators) / 3 { + state.latest_justified_hash = vote.data.target; + state.latest_justified_slot = vote.data.target_slot; + state.justified_slots[vote.data.target_slot] = true; - state.justifications.remove(&vote.target).unwrap(); + state.justifications.remove(&vote.data.target).unwrap(); // Finalization: if the target is the next valid justifiable // hash after the source let mut is_target_next_valid_justifiable_slot = true; - for slot in (vote.source_slot + 1)..vote.target_slot { + for slot in (vote.data.source_slot + 1)..vote.data.target_slot { if is_justifiable_slot(&state.latest_finalized_slot, &slot) { is_target_next_valid_justifiable_slot = false; break; @@ -103,8 +113,8 @@ pub fn process_block(pre_state: &State, block: &Block) -> State { } if is_target_next_valid_justifiable_slot { - state.latest_finalized_hash = vote.source; - state.latest_finalized_slot = vote.source_slot; + state.latest_finalized_hash = vote.data.source; + state.latest_finalized_slot = vote.data.source_slot; } } } @@ -113,7 +123,7 @@ pub fn process_block(pre_state: &State, block: &Block) -> State { } // Get the highest-slot justified block that we know about -pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { +pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { let latest_justified_hash = post_states .values() .max_by_key(|state| state.latest_justified_slot) @@ -127,8 +137,8 @@ pub fn get_latest_justified_hash(post_states: &HashMap) -> Option, provided_root: &Hash, - votes: &Vec, - min_score: u64, + votes: &VariableList, + min_score: usize, ) -> Hash { let mut root = *provided_root; @@ -149,7 +159,8 @@ pub fn get_fork_choice_head( let mut latest_votes = HashMap::::new(); for vote in votes { - latest_votes.insert(vote.data.validator_id, vote.clone()); + let validator_id = vote.data.validator_id; + latest_votes.insert(validator_id, vote.clone()); } // For each block, count the number of votes for that block. A vote diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index d8a796f32..8fadef1bd 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -1,13 +1,13 @@ use ream_pqc::PublicKey; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use std::{ - cell::RefCell, - collections::HashMap, - rc::{Rc, Weak}, +use ssz_types::{ + VariableList, + typenum::{ + U16777216, // 2**24 + }, }; -use tree_hash_derive::TreeHash; +use std::collections::HashMap; use crate::{ block::Block, @@ -17,68 +17,60 @@ use crate::{ is_justifiable_slot, process_block, QueueItem, - state::State, + SLOT_DURATION, + state::LeanState, vote::{Vote, VoteData}, }; -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +// TODO: Add back #[derive(TreeHash)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct Staker { - pub validator_id: u64, + pub validator_id: usize, pub public_key: PublicKey, // Additional to 3SF-mini - pub network: Weak>, pub chain: HashMap, - pub post_states: HashMap, - pub known_votes: Vec, - pub new_votes: Vec, + pub time: usize, // TODO: update the time so on_tick() works properly + // TODO: Add back proper networking instead + // pub network: Weak>, + pub post_states: HashMap, + pub known_votes: VariableList, + pub new_votes: VariableList, pub dependencies: HashMap>, pub genesis_hash: Hash, - pub num_validators: u64, - pub safe_target: Option, + // TODO: Proper validator key handling from static config + pub num_validators: usize, + pub safe_target: Hash, pub head: Hash, } impl Staker { pub fn new( - validator_id: u64, - network: &Rc>, + validator_id: usize, genesis_block: &Block, - genesis_state: &State, + genesis_state: &LeanState, ) -> Staker { let genesis_hash = genesis_block.compute_hash(); let mut chain = HashMap::::new(); chain.insert(genesis_hash, genesis_block.clone()); - let mut post_states = HashMap::::new(); + let mut post_states = HashMap::::new(); post_states.insert(genesis_hash, genesis_state.clone()); Staker { validator_id, public_key: PublicKey {}, - network: Rc::downgrade(network), chain, + time: 0, post_states, - known_votes: Vec::::new(), - new_votes: Vec::::new(), + known_votes: VariableList::::empty(), + new_votes: VariableList::::empty(), dependencies: HashMap::>::new(), genesis_hash, - num_validators: genesis_state.config.num_validators, - safe_target: None, + num_validators: genesis_state.stakers.len(), + safe_target: genesis_hash, head: genesis_hash, } } - /// A helper function that returns the Staker's network reference - /// by abstracting away Reference Counted implementation. - /// - /// Using `self.get_network()` is recommended over using `self.network` directly. - /// - /// Learn more: https://doc.rust-lang.org/std/rc/ - fn get_network(&self) -> Rc> { - self.network // Weak> - .upgrade() // Option>> - .unwrap() // Rc> - } - pub fn latest_justified_hash(&self) -> Option { get_latest_justified_hash(&self.post_states) } @@ -114,11 +106,12 @@ impl Staker { .find(|known_vote| *known_vote == *new_vote) .is_none() { - self.known_votes.push(new_vote.clone()); + // TODO: proper error handling + let _ = self.known_votes.push(new_vote.clone()); } } - self.new_votes.clear(); + self.new_votes = VariableList::empty(); self.recompute_head(); } @@ -130,7 +123,7 @@ impl Staker { // Called every second pub fn tick(&mut self) { - let time_in_slot = self.get_network().borrow().time % SLOT_DURATION; + let time_in_slot = self.time % SLOT_DURATION; // t=0: propose a block if time_in_slot == 0 { @@ -148,15 +141,15 @@ impl Staker { // one honest node receives by this time, every honest node will receive by // the general attestation deadline) } else if time_in_slot == SLOT_DURATION * 2 / 4 { - self.safe_target = Some(self.compute_safe_target()); + self.safe_target = self.compute_safe_target(); // Deadline to accept attestations except for those included in a block } else if time_in_slot == SLOT_DURATION * 3 / 4 { self.accept_new_votes(); } } - fn get_current_slot(&self) -> u64 { - self.get_network().borrow().time / SLOT_DURATION + 2 + fn get_current_slot(&self) -> usize { + self.time / SLOT_DURATION + 2 } // Called when it's the staker's turn to propose a block @@ -172,11 +165,12 @@ impl Staker { let head_state = self.post_states.get(&self.head).unwrap(); let mut new_block = Block { slot: new_slot, + proposer_index: self.validator_id, parent: Some(self.head), - votes: Vec::new(), + votes: VariableList::empty(), state_root: None, }; - let mut state: State; + let mut state: LeanState; // Keep attempt to add valid votes from the list of available votes loop { @@ -199,7 +193,10 @@ impl Staker { break; } - new_block.votes.append(&mut new_votes_to_add); + for vote in new_votes_to_add { + // TODO: proper error handling + let _ = new_block.votes.push(vote); + } } new_block.state_root = Some(state.compute_hash()); @@ -208,21 +205,21 @@ impl Staker { self.chain.insert(new_hash, new_block.clone()); self.post_states.insert(new_hash, state); - self.get_network() - .borrow_mut() - .submit(QueueItem::BlockItem(new_block), self.validator_id); + // TODO: submit to actual network + // self.get_network() + // .borrow_mut() + // .submit(QueueItem::BlockItem(new_block), self.validator_id); } // Called when it's the staker's turn to vote fn vote(&mut self) { let state = self.post_states.get(&self.head).unwrap(); let mut target_block = self.chain.get(&self.head).unwrap(); - let safe_target = self.safe_target.unwrap_or(self.genesis_hash); // If there is no very recent safe target, then vote for the k'th ancestor // of the head for _ in 0..3 { - if target_block.slot > self.chain.get(&safe_target).unwrap().slot { + if target_block.slot > self.chain.get(&self.safe_target).unwrap().slot { target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); } } @@ -259,9 +256,10 @@ impl Staker { self.receive(&QueueItem::VoteItem(vote.clone())); - self.get_network() - .borrow_mut() - .submit(QueueItem::VoteItem(vote), self.validator_id); + // TODO: submit to actual network + // self.get_network() + // .borrow_mut() + // .submit(QueueItem::VoteItem(vote), self.validator_id); } // Called by the p2p network @@ -289,7 +287,8 @@ impl Staker { .find(|known_vote| *known_vote == *vote) .is_none() { - self.known_votes.push(vote.clone()); + // TODO: proper error handling + let _ = self.known_votes.push(vote.clone()); } } @@ -332,7 +331,8 @@ impl Staker { if is_known_vote || is_new_vote { return; } else if self.chain.contains_key(&vote.data.head) { - self.new_votes.push(vote.clone()); + // TODO: proper error handling + let _ = self.new_votes.push(vote.clone()); } else { self.dependencies .entry(vote.data.head) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index e8a28f013..8428767d7 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,29 +1,38 @@ +use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; -use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; +use ssz_types::{ + VariableList, + typenum::{ + U4096, // 2**12 + // U16777216, // 2**24 + }, +}; use std::collections::HashMap; -use tree_hash_derive::TreeHash; use crate::{ - staker::Staker + staker::Staker, Hash, }; -#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +// TODO: Add back #[derive(Encode, Decode, TreeHash)] +#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct LeanState { - pub genesis_time: u64, - pub stakers: VariableList, + pub genesis_time: usize, + pub stakers: VariableList, + pub num_validators: usize, pub latest_justified_hash: Hash, pub latest_justified_slot: usize, pub latest_finalized_hash: Hash, pub latest_finalized_slot: usize, - pub historical_block_hashes: Vec>, - pub justified_slots: Vec, + + pub historical_block_hashes: VariableList, U4096>, + pub justified_slots: VariableList, + pub justifications: HashMap>, } -impl State { +impl LeanState { pub fn compute_hash(&self) -> Hash { let serialized = serde_json::to_string(self).unwrap(); Hash::from_slice(&hash(serialized.as_bytes())) diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index 24232de80..83b51e0a9 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -14,14 +14,14 @@ pub struct Vote { #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct VoteData { - pub validator_id: u64, - pub slot: u64, + pub validator_id: usize, + pub slot: usize, pub head: Hash, - pub head_slot: u64, + pub head_slot: usize, pub target: Hash, - pub target_slot: u64, + pub target_slot: usize, pub source: Hash, - pub source_slot: u64, + pub source_slot: usize, } impl Vote { From 565451ad5b999df48e8d131a4d11ed220955acf2 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 15:07:39 +0700 Subject: [PATCH 03/47] fix: clippy and fmt suggestions --- crates/common/consensus/lean/src/block.rs | 5 +- crates/common/consensus/lean/src/lib.rs | 23 ++++---- crates/common/consensus/lean/src/staker.rs | 63 +++++++--------------- crates/common/consensus/lean/src/state.rs | 6 +-- crates/common/consensus/lean/src/vote.rs | 2 +- 5 files changed, 34 insertions(+), 65 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 5c45d3b26..cc13320e9 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -9,10 +9,7 @@ use ssz_types::{ }, }; -use crate::{ - Hash, - vote::Vote, -}; +use crate::{Hash, vote::Vote}; // TODO: Add back #[derive(TreeHash)] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode)] diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 96696da04..72bacb660 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -1,8 +1,10 @@ pub mod block; -pub mod state; pub mod staker; +pub mod state; pub mod vote; +use std::collections::HashMap; + use alloy_primitives::B256; use serde::{Deserialize, Serialize}; use ssz_types::{ @@ -11,13 +13,8 @@ use ssz_types::{ U16777216, // 2**24 }, }; -use std::collections::HashMap; -use crate::{ - block::Block, - state::LeanState, - vote::Vote, -}; +use crate::{block::Block, state::LeanState, vote::Vote}; pub type Hash = B256; @@ -66,7 +63,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Ignore votes whose source is not already justified, // or whose target is not in the history, or whose target is not a // valid justifiable slot - if state.justified_slots[vote.data.source_slot] == false + if !state.justified_slots[vote.data.source_slot] || Some(vote.data.source) != state.historical_block_hashes[vote.data.source_slot] || Some(vote.data.target) != state.historical_block_hashes[vote.data.target_slot] || vote.data.target_slot <= vote.data.source_slot @@ -124,12 +121,10 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Get the highest-slot justified block that we know about pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { - let latest_justified_hash = post_states + post_states .values() .max_by_key(|state| state.latest_justified_slot) - .map(|state| state.latest_justified_hash); - - latest_justified_hash + .map(|state| state.latest_justified_hash) } // Use LMD GHOST to get the head, given a particular root (usually the @@ -167,7 +162,7 @@ pub fn get_fork_choice_head( // for any descendant of a block also counts as a vote for that block let mut vote_weights = HashMap::::new(); - for (_validator_id, vote) in &latest_votes { + for vote in latest_votes.values() { if blocks.contains_key(&vote.data.head) { let mut block_hash = vote.data.head; while blocks.get(&block_hash).unwrap().slot > blocks.get(&root).unwrap().slot { @@ -209,7 +204,7 @@ pub fn get_fork_choice_head( .max_by_key(|child_hash| { let vote_weight = vote_weights.get(*child_hash).unwrap_or(&0); let slot = blocks.get(*child_hash).unwrap().slot; - (*vote_weight, slot, (*child_hash).clone()) + (*vote_weight, slot, *(*child_hash)) }) .unwrap(); } diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 8fadef1bd..5780cae77 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -1,5 +1,6 @@ -use ream_pqc::PublicKey; -use ream_pqc::PQSignature; +use std::collections::HashMap; + +use ream_pqc::{PQSignature, PublicKey}; use serde::{Deserialize, Serialize}; use ssz_types::{ VariableList, @@ -7,17 +8,11 @@ use ssz_types::{ U16777216, // 2**24 }, }; -use std::collections::HashMap; use crate::{ + Hash, QueueItem, SLOT_DURATION, block::Block, - get_fork_choice_head, - get_latest_justified_hash, - Hash, - is_justifiable_slot, - process_block, - QueueItem, - SLOT_DURATION, + get_fork_choice_head, get_latest_justified_hash, is_justifiable_slot, process_block, state::LeanState, vote::{Vote, VoteData}, }; @@ -43,11 +38,7 @@ pub struct Staker { } impl Staker { - pub fn new( - validator_id: usize, - genesis_block: &Block, - genesis_state: &LeanState, - ) -> Staker { + pub fn new(validator_id: usize, genesis_block: &Block, genesis_state: &LeanState) -> Staker { let genesis_hash = genesis_block.compute_hash(); let mut chain = HashMap::::new(); chain.insert(genesis_hash, genesis_block.clone()); @@ -76,12 +67,9 @@ impl Staker { } pub fn latest_finalized_hash(&self) -> Option { - match self.post_states.get(&self.head) { - Some(state) => { - Some(state.latest_finalized_hash) - }, - None => None - } + self.post_states + .get(&self.head) + .map(|state| state.latest_finalized_hash) } /// Compute the latest block that the staker is allowed to choose as the target @@ -102,10 +90,7 @@ impl Staker { let mut known_votes = self.known_votes.clone().into_iter(); for new_vote in &self.new_votes { - if known_votes - .find(|known_vote| *known_vote == *new_vote) - .is_none() - { + if !known_votes.any(|known_vote| known_vote == *new_vote) { // TODO: proper error handling let _ = self.known_votes.push(new_vote.clone()); } @@ -174,16 +159,12 @@ impl Staker { // Keep attempt to add valid votes from the list of available votes loop { - state = process_block(&head_state, &new_block); + state = process_block(head_state, &new_block); let mut new_votes_to_add = Vec::::new(); for vote in self.known_votes.clone().into_iter() { if vote.data.source == state.latest_justified_hash - && new_block.votes - .clone() - .into_iter() - .find(|v| *v == vote) - .is_none() + && !new_block.votes.clone().into_iter().any(|v| v == vote) { new_votes_to_add.push(vote); } @@ -275,7 +256,7 @@ impl Staker { match self.post_states.get(&block.parent.unwrap()) { Some(parent_state) => { - let state = process_block(&parent_state, &block); + let state = process_block(parent_state, block); self.chain.insert(block_hash, block.clone()); self.post_states.insert(block_hash, state); @@ -283,10 +264,7 @@ impl Staker { let mut known_votes = self.known_votes.clone().into_iter(); for vote in &block.votes { - if known_votes - .find(|known_vote| *known_vote == *vote) - .is_none() - { + if !known_votes.any(|known_vote| known_vote == *vote) { // TODO: proper error handling let _ = self.known_votes.push(vote.clone()); } @@ -309,7 +287,7 @@ impl Staker { // process later once we actually see the parent self.dependencies .entry(block.parent.unwrap()) - .or_insert_with(Vec::new) + .or_default() .push(queue_item.clone()); } } @@ -319,24 +297,23 @@ impl Staker { .known_votes .clone() .into_iter() - .find(|known_vote| known_vote == vote) - .is_some(); + .any(|known_vote| known_vote == *vote); + let is_new_vote = self .new_votes .clone() .into_iter() - .find(|new_vote| new_vote == vote) - .is_some(); + .any(|new_vote| new_vote == *vote); if is_known_vote || is_new_vote { - return; + // Do nothing } else if self.chain.contains_key(&vote.data.head) { // TODO: proper error handling let _ = self.new_votes.push(vote.clone()); } else { self.dependencies .entry(vote.data.head) - .or_insert_with(Vec::new) + .or_default() .push(queue_item.clone()); } } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 8428767d7..1b7bad451 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,17 +1,17 @@ +use std::collections::HashMap; + use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; use ssz_types::{ VariableList, typenum::{ U4096, // 2**12 - // U16777216, // 2**24 }, }; -use std::collections::HashMap; use crate::{ - staker::Staker, Hash, + staker::Staker, }; // TODO: Add back #[derive(Encode, Decode, TreeHash)] diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index 83b51e0a9..d0b6ef86c 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -1,5 +1,5 @@ -use ream_pqc::PQSignature; use ethereum_hashing::hash; +use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use tree_hash_derive::TreeHash; From 34b419abecdc62bce96c5d2df55c16d4b6b83983 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 15:35:11 +0700 Subject: [PATCH 04/47] fix: more linting --- Cargo.lock | 2 +- crates/common/consensus/lean/src/state.rs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c93f6d63..6d2e3493a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5163,7 +5163,6 @@ dependencies = [ "ream-beacon-api-types", "ream-checkpoint-sync", "ream-consensus-beacon", - "ream-consensus-lean", "ream-consensus-misc", "ream-discv5", "ream-executor", @@ -5552,6 +5551,7 @@ dependencies = [ "libp2p-mplex", "parking_lot", "ream-consensus-beacon", + "ream-consensus-lean", "ream-consensus-misc", "ream-discv5", "ream-executor", diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 1b7bad451..a82a6fb9f 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -9,10 +9,7 @@ use ssz_types::{ }, }; -use crate::{ - Hash, - staker::Staker, -}; +use crate::{Hash, staker::Staker}; // TODO: Add back #[derive(Encode, Decode, TreeHash)] #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] From 68795327b716646fdbb9b28966c0cfbe226ed6d6 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 15:48:08 +0700 Subject: [PATCH 05/47] fix: cargo udeps --- bin/ream/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/ream/Cargo.toml b/bin/ream/Cargo.toml index 08843dc13..3564d137c 100644 --- a/bin/ream/Cargo.toml +++ b/bin/ream/Cargo.toml @@ -33,7 +33,6 @@ ream-account-manager.workspace = true ream-beacon-api-types.workspace = true ream-checkpoint-sync.workspace = true ream-consensus-beacon.workspace = true -ream-consensus-lean.workspace = true ream-consensus-misc.workspace = true ream-discv5.workspace = true ream-executor.workspace = true From 17d845ceae8940dcaa6408e6dfe47c4d4e29256b Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna <921194+unnawut@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:30:58 +0700 Subject: [PATCH 06/47] Simplify justifications init Co-authored-by: Jun Song <87601811+syjn99@users.noreply.github.com> --- crates/common/consensus/lean/src/lib.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 72bacb660..6f84a40c5 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -74,12 +74,9 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Track attempts to justify new hashes if !state.justifications.contains_key(&vote.data.target) { - let mut empty_justifications = Vec::::with_capacity(state.num_validators); - empty_justifications.resize(state.num_validators, false); - state .justifications - .insert(vote.data.target, empty_justifications); + .insert(vote.data.target, vec![false; state.num_validators]); } if !state.justifications[&vote.data.target][vote.data.validator_id] { From f9217a2df73583a4c78bf71053900e210280f303 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna <921194+unnawut@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:35:24 +0700 Subject: [PATCH 07/47] Rename Vote and VoteData Co-authored-by: Jun Song <87601811+syjn99@users.noreply.github.com> --- crates/common/consensus/lean/src/vote.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index d0b6ef86c..5458bef18 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -7,13 +7,13 @@ use tree_hash_derive::TreeHash; use crate::Hash; #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct Vote { - pub data: VoteData, +pub struct SignedVote { + pub data: Vote, pub signature: PQSignature, } #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] -pub struct VoteData { +pub struct Vote { pub validator_id: usize, pub slot: usize, pub head: Hash, From e47a432d9879e880c30e045513c250eb26276558 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna <921194+unnawut@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:37:21 +0700 Subject: [PATCH 08/47] Use range any instead of for loop Co-authored-by: Jun Song <87601811+syjn99@users.noreply.github.com> --- crates/common/consensus/lean/src/lib.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 6f84a40c5..7e520545e 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -97,14 +97,9 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Finalization: if the target is the next valid justifiable // hash after the source - let mut is_target_next_valid_justifiable_slot = true; - - for slot in (vote.data.source_slot + 1)..vote.data.target_slot { - if is_justifiable_slot(&state.latest_finalized_slot, &slot) { - is_target_next_valid_justifiable_slot = false; - break; - } - } + let is_target_next_valid_justifiable_slot = !((vote.data.source_slot + 1) + ..vote.data.target_slot) + .any(|slot| is_justifiable_slot(&state.latest_finalized_slot, &slot)); if is_target_next_valid_justifiable_slot { state.latest_finalized_hash = vote.data.source; From 035787429be663cb5a92a3dc226ed090d5ea6283 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 22:42:01 +0700 Subject: [PATCH 09/47] fix: better upsert --- crates/common/consensus/lean/src/lib.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 7e520545e..05993a37b 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -170,14 +170,10 @@ pub fn get_fork_choice_head( for (hash, block) in blocks { if block.parent.is_some() && *vote_weights.get(hash).unwrap_or(&0) >= min_score { - match children_map.get_mut(&block.parent.unwrap()) { - Some(child_hashes) => { - child_hashes.push(*hash); - } - None => { - children_map.insert(block.parent.unwrap(), vec![*hash]); - } - } + children_map + .entry(block.parent.unwrap()) + .or_insert_with(Vec::new) + .push(*hash); } } From f9121e1947e6777ef59270d8a94996a140889faf Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 22:49:20 +0700 Subject: [PATCH 10/47] fix: Vote and SignedVote --- crates/common/consensus/lean/src/lib.rs | 48 +++++++++++----------- crates/common/consensus/lean/src/staker.rs | 18 ++++---- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 05993a37b..755558fa3 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -14,7 +14,7 @@ use ssz_types::{ }, }; -use crate::{block::Block, state::LeanState, vote::Vote}; +use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; pub type Hash = B256; @@ -24,7 +24,7 @@ pub const SLOT_DURATION: usize = 12; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum QueueItem { BlockItem(Block), - VoteItem(Vote), + VoteItem(SignedVote), } // We allow justification of slots either <= 5 or a perfect square or oblong after @@ -63,47 +63,47 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Ignore votes whose source is not already justified, // or whose target is not in the history, or whose target is not a // valid justifiable slot - if !state.justified_slots[vote.data.source_slot] - || Some(vote.data.source) != state.historical_block_hashes[vote.data.source_slot] - || Some(vote.data.target) != state.historical_block_hashes[vote.data.target_slot] - || vote.data.target_slot <= vote.data.source_slot - || !is_justifiable_slot(&state.latest_finalized_slot, &vote.data.target_slot) + if !state.justified_slots[vote.source_slot] + || Some(vote.source) != state.historical_block_hashes[vote.source_slot] + || Some(vote.target) != state.historical_block_hashes[vote.target_slot] + || vote.target_slot <= vote.source_slot + || !is_justifiable_slot(&state.latest_finalized_slot, &vote.target_slot) { continue; } // Track attempts to justify new hashes - if !state.justifications.contains_key(&vote.data.target) { + if !state.justifications.contains_key(&vote.target) { state .justifications - .insert(vote.data.target, vec![false; state.num_validators]); + .insert(vote.target, vec![false; state.num_validators]); } - if !state.justifications[&vote.data.target][vote.data.validator_id] { - state.justifications.get_mut(&vote.data.target).unwrap()[vote.data.validator_id] = true; + if !state.justifications[&vote.target][vote.validator_id] { + state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id] = true; } - let count = state.justifications[&vote.data.target] + let count = state.justifications[&vote.target] .iter() .fold(0, |sum, justification| sum + *justification as usize); // If 2/3 voted for the same new valid hash to justify if count == (2 * state.num_validators) / 3 { - state.latest_justified_hash = vote.data.target; - state.latest_justified_slot = vote.data.target_slot; - state.justified_slots[vote.data.target_slot] = true; + state.latest_justified_hash = vote.target; + state.latest_justified_slot = vote.target_slot; + state.justified_slots[vote.target_slot] = true; - state.justifications.remove(&vote.data.target).unwrap(); + state.justifications.remove(&vote.target).unwrap(); // Finalization: if the target is the next valid justifiable // hash after the source - let is_target_next_valid_justifiable_slot = !((vote.data.source_slot + 1) - ..vote.data.target_slot) + let is_target_next_valid_justifiable_slot = !((vote.source_slot + 1) + ..vote.target_slot) .any(|slot| is_justifiable_slot(&state.latest_finalized_slot, &slot)); if is_target_next_valid_justifiable_slot { - state.latest_finalized_hash = vote.data.source; - state.latest_finalized_slot = vote.data.source_slot; + state.latest_finalized_hash = vote.source; + state.latest_finalized_slot = vote.source_slot; } } } @@ -140,13 +140,13 @@ pub fn get_fork_choice_head( // Sort votes by ascending slots to ensure that new votes are inserted last let mut sorted_votes = votes.clone(); - sorted_votes.sort_by_key(|vote| vote.data.slot); + sorted_votes.sort_by_key(|vote| vote.slot); // Prepare a map of validator_id -> their vote let mut latest_votes = HashMap::::new(); for vote in votes { - let validator_id = vote.data.validator_id; + let validator_id = vote.validator_id; latest_votes.insert(validator_id, vote.clone()); } @@ -155,8 +155,8 @@ pub fn get_fork_choice_head( let mut vote_weights = HashMap::::new(); for vote in latest_votes.values() { - if blocks.contains_key(&vote.data.head) { - let mut block_hash = vote.data.head; + if blocks.contains_key(&vote.head) { + let mut block_hash = vote.head; while blocks.get(&block_hash).unwrap().slot > blocks.get(&root).unwrap().slot { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); vote_weights.insert(block_hash, current_weights + 1); diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 5780cae77..0b15ce1c5 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -14,7 +14,7 @@ use crate::{ block::Block, get_fork_choice_head, get_latest_justified_hash, is_justifiable_slot, process_block, state::LeanState, - vote::{Vote, VoteData}, + vote::{SignedVote, Vote}, }; // TODO: Add back #[derive(TreeHash)] @@ -163,7 +163,7 @@ impl Staker { let mut new_votes_to_add = Vec::::new(); for vote in self.known_votes.clone().into_iter() { - if vote.data.source == state.latest_justified_hash + if vote.source == state.latest_justified_hash && !new_block.votes.clone().into_iter().any(|v| v == vote) { new_votes_to_add.push(vote); @@ -211,7 +211,7 @@ impl Staker { target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); } - let vote_data = VoteData { + let vote = Vote { validator_id: self.validator_id, slot: self.get_current_slot(), head: self.head, @@ -222,8 +222,8 @@ impl Staker { source_slot: state.latest_justified_slot, }; - let vote = Vote { - data: vote_data, + let signed_vote = SignedVote { + data: vote, signature: PQSignature {}, }; @@ -235,7 +235,7 @@ impl Staker { &state.latest_justified_slot ); - self.receive(&QueueItem::VoteItem(vote.clone())); + self.receive(&QueueItem::VoteItem(signed_vote.clone())); // TODO: submit to actual network // self.get_network() @@ -297,19 +297,19 @@ impl Staker { .known_votes .clone() .into_iter() - .any(|known_vote| known_vote == *vote); + .any(|known_vote| known_vote == vote.data); let is_new_vote = self .new_votes .clone() .into_iter() - .any(|new_vote| new_vote == *vote); + .any(|new_vote| new_vote == vote.data); if is_known_vote || is_new_vote { // Do nothing } else if self.chain.contains_key(&vote.data.head) { // TODO: proper error handling - let _ = self.new_votes.push(vote.clone()); + let _ = self.new_votes.push(vote.data.clone()); } else { self.dependencies .entry(vote.data.head) From e1fac7b5c516277ad98ee79610bf1e80f31349e8 Mon Sep 17 00:00:00 2001 From: unnawut Date: Thu, 31 Jul 2025 23:00:38 +0700 Subject: [PATCH 11/47] docs: change todo from adding TreeHash to splitting the Staker and service --- crates/common/consensus/lean/src/staker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 0b15ce1c5..52f282791 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -17,7 +17,7 @@ use crate::{ vote::{SignedVote, Vote}, }; -// TODO: Add back #[derive(TreeHash)] +// TODO: Split to Staker and Service #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct Staker { pub validator_id: usize, From 486ab5bdc947d8fa33af1f40990914527c6f1d40 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 10:55:03 +0700 Subject: [PATCH 12/47] refactor: use B256 directly --- crates/common/consensus/lean/src/block.rs | 11 ++++----- crates/common/consensus/lean/src/lib.rs | 19 +++++++--------- crates/common/consensus/lean/src/staker.rs | 26 ++++++++++++---------- crates/common/consensus/lean/src/state.rs | 14 ++++++------ crates/common/consensus/lean/src/vote.rs | 13 +++++------ 5 files changed, 41 insertions(+), 42 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index cc13320e9..96b67d43f 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -1,3 +1,4 @@ +use alloy_primitives::B256; use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; @@ -9,7 +10,7 @@ use ssz_types::{ }, }; -use crate::{Hash, vote::Vote}; +use crate::vote::Vote; // TODO: Add back #[derive(TreeHash)] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode)] @@ -23,14 +24,14 @@ pub struct SignedBlock { pub struct Block { pub slot: usize, pub proposer_index: usize, - pub parent: Option, + pub parent: Option, pub votes: VariableList, - pub state_root: Option, + pub state_root: Option, } impl Block { - pub fn compute_hash(&self) -> Hash { + pub fn compute_hash(&self) -> B256 { let serialized = serde_json::to_string(self).unwrap(); - Hash::from_slice(&hash(serialized.as_bytes())) + B256::from_slice(&hash(serialized.as_bytes())) } } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 755558fa3..166a6c2ec 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -16,9 +16,6 @@ use ssz_types::{ use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; -pub type Hash = B256; - -pub const ZERO_HASH: Hash = Hash::ZERO; pub const SLOT_DURATION: usize = 12; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] @@ -112,7 +109,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { } // Get the highest-slot justified block that we know about -pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { +pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { post_states .values() .max_by_key(|state| state.latest_justified_slot) @@ -122,15 +119,15 @@ pub fn get_latest_justified_hash(post_states: &HashMap) -> Opti // Use LMD GHOST to get the head, given a particular root (usually the // latest known justified block) pub fn get_fork_choice_head( - blocks: &HashMap, - provided_root: &Hash, - votes: &VariableList, + blocks: &HashMap, + provided_root: &B256, + votes: &VariableList, min_score: usize, -) -> Hash { +) -> B256 { let mut root = *provided_root; // Start at genesis by default - if *root == ZERO_HASH { + if *root == B256::ZERO { root = blocks .iter() .min_by_key(|(_, block)| block.slot) @@ -152,7 +149,7 @@ pub fn get_fork_choice_head( // For each block, count the number of votes for that block. A vote // for any descendant of a block also counts as a vote for that block - let mut vote_weights = HashMap::::new(); + let mut vote_weights = HashMap::::new(); for vote in latest_votes.values() { if blocks.contains_key(&vote.head) { @@ -166,7 +163,7 @@ pub fn get_fork_choice_head( } // Identify the children of each block - let mut children_map = HashMap::>::new(); + let mut children_map = HashMap::>::new(); for (hash, block) in blocks { if block.parent.is_some() && *vote_weights.get(hash).unwrap_or(&0) >= min_score { diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 52f282791..1ade03061 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use alloy_primitives::B256; use ream_pqc::{PQSignature, PublicKey}; use serde::{Deserialize, Serialize}; use ssz_types::{ @@ -10,7 +11,7 @@ use ssz_types::{ }; use crate::{ - Hash, QueueItem, SLOT_DURATION, + QueueItem, SLOT_DURATION, block::Block, get_fork_choice_head, get_latest_justified_hash, is_justifiable_slot, process_block, state::LeanState, @@ -22,28 +23,28 @@ use crate::{ pub struct Staker { pub validator_id: usize, pub public_key: PublicKey, // Additional to 3SF-mini - pub chain: HashMap, + pub chain: HashMap, pub time: usize, // TODO: update the time so on_tick() works properly // TODO: Add back proper networking instead // pub network: Weak>, - pub post_states: HashMap, pub known_votes: VariableList, pub new_votes: VariableList, - pub dependencies: HashMap>, - pub genesis_hash: Hash, + pub post_states: HashMap, + pub dependencies: HashMap>, + pub genesis_hash: B256, // TODO: Proper validator key handling from static config pub num_validators: usize, - pub safe_target: Hash, - pub head: Hash, + pub safe_target: B256, + pub head: B256, } impl Staker { pub fn new(validator_id: usize, genesis_block: &Block, genesis_state: &LeanState) -> Staker { let genesis_hash = genesis_block.compute_hash(); - let mut chain = HashMap::::new(); + let mut chain = HashMap::::new(); chain.insert(genesis_hash, genesis_block.clone()); - let mut post_states = HashMap::::new(); + let mut post_states = HashMap::::new(); post_states.insert(genesis_hash, genesis_state.clone()); Staker { @@ -55,6 +56,7 @@ impl Staker { known_votes: VariableList::::empty(), new_votes: VariableList::::empty(), dependencies: HashMap::>::new(), + dependencies: HashMap::>::new(), genesis_hash, num_validators: genesis_state.stakers.len(), safe_target: genesis_hash, @@ -62,18 +64,18 @@ impl Staker { } } - pub fn latest_justified_hash(&self) -> Option { + pub fn latest_justified_hash(&self) -> Option { get_latest_justified_hash(&self.post_states) } - pub fn latest_finalized_hash(&self) -> Option { + pub fn latest_finalized_hash(&self) -> Option { self.post_states .get(&self.head) .map(|state| state.latest_finalized_hash) } /// Compute the latest block that the staker is allowed to choose as the target - fn compute_safe_target(&self) -> Hash { + fn compute_safe_target(&self) -> B256 { let justified_hash = get_latest_justified_hash(&self.post_states).unwrap(); get_fork_choice_head( diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index a82a6fb9f..5fe996065 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use alloy_primitives::B256; use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; use ssz_types::{ @@ -9,7 +10,6 @@ use ssz_types::{ }, }; -use crate::{Hash, staker::Staker}; // TODO: Add back #[derive(Encode, Decode, TreeHash)] #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] @@ -18,20 +18,20 @@ pub struct LeanState { pub stakers: VariableList, pub num_validators: usize, - pub latest_justified_hash: Hash, + pub latest_justified_hash: B256, pub latest_justified_slot: usize, - pub latest_finalized_hash: Hash, + pub latest_finalized_hash: B256, pub latest_finalized_slot: usize, - pub historical_block_hashes: VariableList, U4096>, + pub historical_block_hashes: VariableList, U4096>, pub justified_slots: VariableList, - pub justifications: HashMap>, + pub justifications: HashMap>, } impl LeanState { - pub fn compute_hash(&self) -> Hash { + pub fn compute_hash(&self) -> B256 { let serialized = serde_json::to_string(self).unwrap(); - Hash::from_slice(&hash(serialized.as_bytes())) + B256::from_slice(&hash(serialized.as_bytes())) } } diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index 5458bef18..ba1d7a352 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -1,11 +1,10 @@ +use alloy_primitives::B256; use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use tree_hash_derive::TreeHash; -use crate::Hash; - #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct SignedVote { pub data: Vote, @@ -16,17 +15,17 @@ pub struct SignedVote { pub struct Vote { pub validator_id: usize, pub slot: usize, - pub head: Hash, + pub head: B256, pub head_slot: usize, - pub target: Hash, + pub target: B256, pub target_slot: usize, - pub source: Hash, + pub source: B256, pub source_slot: usize, } impl Vote { - pub fn compute_hash(&self) -> Hash { + pub fn compute_hash(&self) -> B256 { let serialized = serde_json::to_string(self).unwrap(); - Hash::from_slice(&hash(serialized.as_bytes())) + B256::from_slice(&hash(serialized.as_bytes())) } } From 306b951742cff1a2ab517fd9e2db51c1a30952cb Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 10:56:59 +0700 Subject: [PATCH 13/47] refactor: bring back Config and use U4096 for both max blocks and validators --- crates/common/consensus/lean/src/config.rs | 8 ++++++++ crates/common/consensus/lean/src/lib.rs | 15 +++++---------- crates/common/consensus/lean/src/staker.rs | 21 +++++++-------------- crates/common/consensus/lean/src/state.rs | 15 ++++----------- 4 files changed, 24 insertions(+), 35 deletions(-) create mode 100644 crates/common/consensus/lean/src/config.rs diff --git a/crates/common/consensus/lean/src/config.rs b/crates/common/consensus/lean/src/config.rs new file mode 100644 index 000000000..8a79122aa --- /dev/null +++ b/crates/common/consensus/lean/src/config.rs @@ -0,0 +1,8 @@ +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use tree_hash_derive::TreeHash; + +#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +pub struct Config { + pub num_validators: usize, +} diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 166a6c2ec..c4b3e3c2e 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -1,18 +1,13 @@ pub mod block; +pub mod config; pub mod staker; pub mod state; pub mod vote; -use std::collections::HashMap; - use alloy_primitives::B256; use serde::{Deserialize, Serialize}; -use ssz_types::{ - VariableList, - typenum::{ - U16777216, // 2**24 - }, -}; +use ssz_types::{typenum::U4096, VariableList}; +use std::collections::HashMap; use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; @@ -73,7 +68,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { if !state.justifications.contains_key(&vote.target) { state .justifications - .insert(vote.target, vec![false; state.num_validators]); + .insert(vote.target, vec![false; state.config.num_validators]); } if !state.justifications[&vote.target][vote.validator_id] { @@ -85,7 +80,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { .fold(0, |sum, justification| sum + *justification as usize); // If 2/3 voted for the same new valid hash to justify - if count == (2 * state.num_validators) / 3 { + if count == (2 * state.config.num_validators) / 3 { state.latest_justified_hash = vote.target; state.latest_justified_slot = vote.target_slot; state.justified_slots[vote.target_slot] = true; diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 1ade03061..d7e67a563 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -1,14 +1,8 @@ -use std::collections::HashMap; - use alloy_primitives::B256; use ream_pqc::{PQSignature, PublicKey}; use serde::{Deserialize, Serialize}; -use ssz_types::{ - VariableList, - typenum::{ - U16777216, // 2**24 - }, -}; +use ssz_types::{typenum::U4096, VariableList}; +use std::collections::HashMap; use crate::{ QueueItem, SLOT_DURATION, @@ -27,9 +21,9 @@ pub struct Staker { pub time: usize, // TODO: update the time so on_tick() works properly // TODO: Add back proper networking instead // pub network: Weak>, - pub known_votes: VariableList, - pub new_votes: VariableList, pub post_states: HashMap, + pub known_votes: VariableList, + pub new_votes: VariableList, pub dependencies: HashMap>, pub genesis_hash: B256, // TODO: Proper validator key handling from static config @@ -53,12 +47,11 @@ impl Staker { chain, time: 0, post_states, - known_votes: VariableList::::empty(), - new_votes: VariableList::::empty(), - dependencies: HashMap::>::new(), + known_votes: VariableList::::empty(), + new_votes: VariableList::::empty(), dependencies: HashMap::>::new(), genesis_hash, - num_validators: genesis_state.stakers.len(), + num_validators: genesis_state.config.num_validators, safe_target: genesis_hash, head: genesis_hash, } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 5fe996065..20fd53ff0 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,22 +1,15 @@ -use std::collections::HashMap; - use alloy_primitives::B256; use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; -use ssz_types::{ - VariableList, - typenum::{ - U4096, // 2**12 - }, -}; +use ssz_types::{typenum::U4096, VariableList}; +use std::collections::HashMap; +use crate::config::Config; // TODO: Add back #[derive(Encode, Decode, TreeHash)] #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] pub struct LeanState { - pub genesis_time: usize, - pub stakers: VariableList, - pub num_validators: usize, + pub config: Config, pub latest_justified_hash: B256, pub latest_justified_slot: usize, From dc49f5cd7aceaee98a97c659381cfc30660b4213 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 11:43:52 +0700 Subject: [PATCH 14/47] refactor: usize to u64 --- crates/common/consensus/lean/src/block.rs | 4 ++-- crates/common/consensus/lean/src/config.rs | 2 +- crates/common/consensus/lean/src/lib.rs | 28 +++++++++++----------- crates/common/consensus/lean/src/staker.rs | 10 ++++---- crates/common/consensus/lean/src/state.rs | 4 ++-- crates/common/consensus/lean/src/vote.rs | 10 ++++---- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 96b67d43f..61ba21531 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -22,8 +22,8 @@ pub struct SignedBlock { // TODO: Add back #[derive(TreeHash)] #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, Default)] pub struct Block { - pub slot: usize, - pub proposer_index: usize, + pub slot: u64, + pub proposer_index: u64, pub parent: Option, pub votes: VariableList, pub state_root: Option, diff --git a/crates/common/consensus/lean/src/config.rs b/crates/common/consensus/lean/src/config.rs index 8a79122aa..82ed0def5 100644 --- a/crates/common/consensus/lean/src/config.rs +++ b/crates/common/consensus/lean/src/config.rs @@ -4,5 +4,5 @@ use tree_hash_derive::TreeHash; #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct Config { - pub num_validators: usize, + pub num_validators: u64, } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index c4b3e3c2e..e9e22999a 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; -pub const SLOT_DURATION: usize = 12; +pub const SLOT_DURATION: u64 = 12; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum QueueItem { @@ -22,7 +22,7 @@ pub enum QueueItem { // We allow justification of slots either <= 5 or a perfect square or oblong after // the latest finalized slot. This gives us a backoff technique and ensures // finality keeps progressing even under high latency -pub fn is_justifiable_slot(finalized_slot: &usize, candidate_slot: &usize) -> bool { +pub fn is_justifiable_slot(finalized_slot: &u64, candidate_slot: &u64) -> bool { assert!( candidate_slot >= finalized_slot, "Candidate slot ({candidate_slot}) is less than finalized slot ({finalized_slot})" @@ -44,7 +44,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { let _ = state.historical_block_hashes.push(block.parent); let _ = state.justified_slots.push(false); - while state.historical_block_hashes.len() < block.slot { + while state.historical_block_hashes.len() < block.slot as usize { // TODO: proper error handlings let _ = state.justified_slots.push(false); let _ = state.historical_block_hashes.push(None); @@ -55,9 +55,9 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Ignore votes whose source is not already justified, // or whose target is not in the history, or whose target is not a // valid justifiable slot - if !state.justified_slots[vote.source_slot] - || Some(vote.source) != state.historical_block_hashes[vote.source_slot] - || Some(vote.target) != state.historical_block_hashes[vote.target_slot] + if !state.justified_slots[vote.source_slot as usize] + || Some(vote.source) != state.historical_block_hashes[vote.source_slot as usize] + || Some(vote.target) != state.historical_block_hashes[vote.target_slot as usize] || vote.target_slot <= vote.source_slot || !is_justifiable_slot(&state.latest_finalized_slot, &vote.target_slot) { @@ -68,11 +68,11 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { if !state.justifications.contains_key(&vote.target) { state .justifications - .insert(vote.target, vec![false; state.config.num_validators]); + .insert(vote.target, vec![false; state.config.num_validators as usize]); } - if !state.justifications[&vote.target][vote.validator_id] { - state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id] = true; + if !state.justifications[&vote.target][vote.validator_id as usize] { + state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id as usize] = true; } let count = state.justifications[&vote.target] @@ -80,10 +80,10 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { .fold(0, |sum, justification| sum + *justification as usize); // If 2/3 voted for the same new valid hash to justify - if count == (2 * state.config.num_validators) / 3 { + if count == (2 * state.config.num_validators as usize) / 3 { state.latest_justified_hash = vote.target; state.latest_justified_slot = vote.target_slot; - state.justified_slots[vote.target_slot] = true; + state.justified_slots[vote.target_slot as usize] = true; state.justifications.remove(&vote.target).unwrap(); @@ -117,7 +117,7 @@ pub fn get_fork_choice_head( blocks: &HashMap, provided_root: &B256, votes: &VariableList, - min_score: usize, + min_score: u64, ) -> B256 { let mut root = *provided_root; @@ -135,7 +135,7 @@ pub fn get_fork_choice_head( sorted_votes.sort_by_key(|vote| vote.slot); // Prepare a map of validator_id -> their vote - let mut latest_votes = HashMap::::new(); + let mut latest_votes = HashMap::::new(); for vote in votes { let validator_id = vote.validator_id; @@ -144,7 +144,7 @@ pub fn get_fork_choice_head( // For each block, count the number of votes for that block. A vote // for any descendant of a block also counts as a vote for that block - let mut vote_weights = HashMap::::new(); + let mut vote_weights = HashMap::::new(); for vote in latest_votes.values() { if blocks.contains_key(&vote.head) { diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index d7e67a563..1808d4638 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -15,10 +15,10 @@ use crate::{ // TODO: Split to Staker and Service #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct Staker { - pub validator_id: usize, + pub validator_id: u64, pub public_key: PublicKey, // Additional to 3SF-mini pub chain: HashMap, - pub time: usize, // TODO: update the time so on_tick() works properly + pub time: u64, // TODO: update the time so on_tick() works properly // TODO: Add back proper networking instead // pub network: Weak>, pub post_states: HashMap, @@ -27,13 +27,13 @@ pub struct Staker { pub dependencies: HashMap>, pub genesis_hash: B256, // TODO: Proper validator key handling from static config - pub num_validators: usize, + pub num_validators: u64, pub safe_target: B256, pub head: B256, } impl Staker { - pub fn new(validator_id: usize, genesis_block: &Block, genesis_state: &LeanState) -> Staker { + pub fn new(validator_id: u64, genesis_block: &Block, genesis_state: &LeanState) -> Staker { let genesis_hash = genesis_block.compute_hash(); let mut chain = HashMap::::new(); chain.insert(genesis_hash, genesis_block.clone()); @@ -128,7 +128,7 @@ impl Staker { } } - fn get_current_slot(&self) -> usize { + fn get_current_slot(&self) -> u64 { self.time / SLOT_DURATION + 2 } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 20fd53ff0..07913e9cb 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -12,9 +12,9 @@ pub struct LeanState { pub config: Config, pub latest_justified_hash: B256, - pub latest_justified_slot: usize, + pub latest_justified_slot: u64, pub latest_finalized_hash: B256, - pub latest_finalized_slot: usize, + pub latest_finalized_slot: u64, pub historical_block_hashes: VariableList, U4096>, pub justified_slots: VariableList, diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index ba1d7a352..ac3be2a01 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -13,14 +13,14 @@ pub struct SignedVote { #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct Vote { - pub validator_id: usize, - pub slot: usize, + pub validator_id: u64, + pub slot: u64, pub head: B256, - pub head_slot: usize, + pub head_slot: u64, pub target: B256, - pub target_slot: usize, + pub target_slot: u64, pub source: B256, - pub source_slot: usize, + pub source_slot: u64, } impl Vote { From b7b33476f8d1bac8286936f89f2668e95c5c163e Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 11:45:06 +0700 Subject: [PATCH 15/47] fix: remove proposer_index from block --- crates/common/consensus/lean/src/block.rs | 1 - crates/common/consensus/lean/src/staker.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 61ba21531..5dd71ddc1 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -23,7 +23,6 @@ pub struct SignedBlock { #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, Default)] pub struct Block { pub slot: u64, - pub proposer_index: u64, pub parent: Option, pub votes: VariableList, pub state_root: Option, diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 1808d4638..c1a8bb6f1 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -145,7 +145,6 @@ impl Staker { let head_state = self.post_states.get(&self.head).unwrap(); let mut new_block = Block { slot: new_slot, - proposer_index: self.validator_id, parent: Some(self.head), votes: VariableList::empty(), state_root: None, From 3cde64a6cece66544545fcbef69789f2bb2e7605 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 11:52:59 +0700 Subject: [PATCH 16/47] refactor: replace let _ with expect() --- crates/common/consensus/lean/src/lib.rs | 10 ++++------ crates/common/consensus/lean/src/staker.rs | 11 ++++------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index e9e22999a..dca649daa 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -40,14 +40,12 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { let mut state = pre_state.clone(); // Track historical blocks in the state - // TODO: proper error handlings - let _ = state.historical_block_hashes.push(block.parent); - let _ = state.justified_slots.push(false); + state.historical_block_hashes.push(block.parent).expect("Failed to add block.parent to historical_block_hashes"); + state.justified_slots.push(false).expect("Failed to add to justified_slots"); while state.historical_block_hashes.len() < block.slot as usize { - // TODO: proper error handlings - let _ = state.justified_slots.push(false); - let _ = state.historical_block_hashes.push(None); + state.justified_slots.push(false).expect("Failed to prefill justified_slots"); + state.historical_block_hashes.push(None).expect("Failed to prefill historical_block_hashes"); } // Process votes diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index c1a8bb6f1..e5150db24 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -86,8 +86,7 @@ impl Staker { for new_vote in &self.new_votes { if !known_votes.any(|known_vote| known_vote == *new_vote) { - // TODO: proper error handling - let _ = self.known_votes.push(new_vote.clone()); + self.known_votes.push(new_vote.clone()).expect("Failed to push new_vote to known_votes"); } } @@ -170,7 +169,7 @@ impl Staker { for vote in new_votes_to_add { // TODO: proper error handling - let _ = new_block.votes.push(vote); + new_block.votes.push(vote).expect("Failed to add vote to new_block"); } } @@ -259,8 +258,7 @@ impl Staker { for vote in &block.votes { if !known_votes.any(|known_vote| known_vote == *vote) { - // TODO: proper error handling - let _ = self.known_votes.push(vote.clone()); + self.known_votes.push(vote.clone()).expect("Failed to push vote to known_votes"); } } @@ -302,8 +300,7 @@ impl Staker { if is_known_vote || is_new_vote { // Do nothing } else if self.chain.contains_key(&vote.data.head) { - // TODO: proper error handling - let _ = self.new_votes.push(vote.data.clone()); + self.new_votes.push(vote.data.clone()).expect("Failed to push vote to new_votes"); } else { self.dependencies .entry(vote.data.head) From e637f4bc94456673d1254df7cb029f6076cd076e Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 14:07:07 +0700 Subject: [PATCH 17/47] fix: unwrap to expect --- crates/common/consensus/lean/src/staker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index e5150db24..b653ca0fe 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -96,7 +96,7 @@ impl Staker { // Done upon processing new votes or a new block fn recompute_head(&mut self) { - let justified_hash = get_latest_justified_hash(&self.post_states).unwrap(); + let justified_hash = get_latest_justified_hash(&self.post_states).expect("Failed to get latest_justified_hash from post_states"); self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0); } From 3805ccbc5270f29e18707ea2f984823240650527 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 14:15:06 +0700 Subject: [PATCH 18/47] fix: mis-implementation of latest_votes --- crates/common/consensus/lean/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index dca649daa..4361b79b5 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -128,6 +128,8 @@ pub fn get_fork_choice_head( .unwrap(); } + // Identify latest votes + // Sort votes by ascending slots to ensure that new votes are inserted last let mut sorted_votes = votes.clone(); sorted_votes.sort_by_key(|vote| vote.slot); @@ -135,7 +137,7 @@ pub fn get_fork_choice_head( // Prepare a map of validator_id -> their vote let mut latest_votes = HashMap::::new(); - for vote in votes { + for vote in sorted_votes { let validator_id = vote.validator_id; latest_votes.insert(validator_id, vote.clone()); } From 29562bbe8c569cebba537deb03ca6d2a0cbc6765 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 14:22:01 +0700 Subject: [PATCH 19/47] fix: Staker instantiation --- crates/common/consensus/lean/src/staker.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index b653ca0fe..099a0eafe 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -33,27 +33,22 @@ pub struct Staker { } impl Staker { - pub fn new(validator_id: u64, genesis_block: &Block, genesis_state: &LeanState) -> Staker { + pub fn new(validator_id: u64, genesis_block: Block, genesis_state: LeanState) -> Staker { let genesis_hash = genesis_block.compute_hash(); - let mut chain = HashMap::::new(); - chain.insert(genesis_hash, genesis_block.clone()); - - let mut post_states = HashMap::::new(); - post_states.insert(genesis_hash, genesis_state.clone()); Staker { validator_id, public_key: PublicKey {}, - chain, time: 0, - post_states, - known_votes: VariableList::::empty(), - new_votes: VariableList::::empty(), - dependencies: HashMap::>::new(), + known_votes: VariableList::empty(), + new_votes: VariableList::empty(), + dependencies: HashMap::new(), genesis_hash, num_validators: genesis_state.config.num_validators, safe_target: genesis_hash, head: genesis_hash, + chain: HashMap::from([(genesis_hash, genesis_block)]), + post_states: HashMap::from([(genesis_hash, genesis_state)]), } } From 9c31cc3e4e0e7542c8599321e760e84fb963b484 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 14:23:52 +0700 Subject: [PATCH 20/47] fix: change Weak> to Arc> --- crates/common/consensus/lean/src/staker.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/consensus/lean/src/staker.rs index 099a0eafe..406e3d23c 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/consensus/lean/src/staker.rs @@ -20,7 +20,7 @@ pub struct Staker { pub chain: HashMap, pub time: u64, // TODO: update the time so on_tick() works properly // TODO: Add back proper networking instead - // pub network: Weak>, + // pub network: Arc>, pub post_states: HashMap, pub known_votes: VariableList, pub new_votes: VariableList, From 809a3dbd67145bf5ee10abf069eb3632bcf7e8d1 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:01:57 +0700 Subject: [PATCH 21/47] feat: move Staker from common/consensus/lean to common/lean_chain --- Cargo.lock | 16 +++++++ Cargo.toml | 2 + crates/common/consensus/lean/src/lib.rs | 4 +- crates/common/lean_chain/Cargo.toml | 23 ++++++++++ crates/common/lean_chain/src/lib.rs | 1 + .../lean => lean_chain}/src/staker.rs | 43 ++++++++----------- crates/networking/manager/Cargo.toml | 1 + crates/networking/p2p/Cargo.toml | 1 + crates/networking/p2p/src/network/lean.rs | 8 +++- 9 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 crates/common/lean_chain/Cargo.toml create mode 100644 crates/common/lean_chain/src/lib.rs rename crates/common/{consensus/lean => lean_chain}/src/staker.rs (88%) diff --git a/Cargo.lock b/Cargo.lock index 6d2e3493a..9410b8a0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5443,6 +5443,21 @@ dependencies = [ "ssz_types", ] +[[package]] +name = "ream-lean-chain" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "ream-consensus-lean", + "ream-network-spec", + "ream-p2p", + "ream-pqc", + "serde", + "ssz_types", + "tokio", + "tracing", +] + [[package]] name = "ream-light-client" version = "0.1.0" @@ -5488,6 +5503,7 @@ dependencies = [ "ream-execution-engine", "ream-executor", "ream-fork-choice", + "ream-lean-chain", "ream-network-spec", "ream-operation-pool", "ream-p2p", diff --git a/Cargo.toml b/Cargo.toml index d6ac4fcac..fb2471680 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/common/execution_engine", "crates/common/executor", "crates/common/fork_choice", + "crates/common/lean_chain", "crates/common/light_client", "crates/common/network_spec", "crates/common/node", @@ -126,6 +127,7 @@ ream-execution-engine = { path = "crates/common/execution_engine" } ream-executor = { path = "crates/common/executor" } ream-fork-choice = { path = "crates/common/fork_choice" } ream-keystore = { path = "crates/crypto/keystore" } +ream-lean-chain = { path = "crates/common/lean_chain" } ream-light-client = { path = "crates/common/light_client" } ream-merkle = { path = "crates/crypto/merkle" } ream-network-manager = { path = "crates/networking/manager" } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 4361b79b5..e6a3f1cdf 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -1,12 +1,10 @@ pub mod block; pub mod config; -pub mod staker; pub mod state; pub mod vote; use alloy_primitives::B256; use serde::{Deserialize, Serialize}; -use ssz_types::{typenum::U4096, VariableList}; use std::collections::HashMap; use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; @@ -114,7 +112,7 @@ pub fn get_latest_justified_hash(post_states: &HashMap) -> Opti pub fn get_fork_choice_head( blocks: &HashMap, provided_root: &B256, - votes: &VariableList, + votes: &Vec, min_score: u64, ) -> B256 { let mut root = *provided_root; diff --git a/crates/common/lean_chain/Cargo.toml b/crates/common/lean_chain/Cargo.toml new file mode 100644 index 000000000..d29715a1b --- /dev/null +++ b/crates/common/lean_chain/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "ream-lean-chain" +authors.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +alloy-primitives.workspace = true +serde.workspace = true +ssz_types.workspace = true +tokio.workspace = true +tracing.workspace = true + +# ream dependencies +ream-consensus-lean.workspace = true +ream-network-spec.workspace = true +ream-p2p.workspace = true +ream-pqc.workspace = true diff --git a/crates/common/lean_chain/src/lib.rs b/crates/common/lean_chain/src/lib.rs new file mode 100644 index 000000000..c8f122945 --- /dev/null +++ b/crates/common/lean_chain/src/lib.rs @@ -0,0 +1 @@ +pub mod staker; diff --git a/crates/common/consensus/lean/src/staker.rs b/crates/common/lean_chain/src/staker.rs similarity index 88% rename from crates/common/consensus/lean/src/staker.rs rename to crates/common/lean_chain/src/staker.rs index 406e3d23c..c80e58ce5 100644 --- a/crates/common/consensus/lean/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -1,10 +1,10 @@ use alloy_primitives::B256; -use ream_pqc::{PQSignature, PublicKey}; -use serde::{Deserialize, Serialize}; -use ssz_types::{typenum::U4096, VariableList}; +use ream_p2p::network::lean::NetworkService; +use ream_pqc::PQSignature; +use ssz_types::VariableList; use std::collections::HashMap; - -use crate::{ +use std::sync::{Arc, Mutex}; +use ream_consensus_lean::{ QueueItem, SLOT_DURATION, block::Block, get_fork_choice_head, get_latest_justified_hash, is_justifiable_slot, process_block, @@ -12,36 +12,29 @@ use crate::{ vote::{SignedVote, Vote}, }; -// TODO: Split to Staker and Service -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct Staker { pub validator_id: u64, - pub public_key: PublicKey, // Additional to 3SF-mini pub chain: HashMap, - pub time: u64, // TODO: update the time so on_tick() works properly - // TODO: Add back proper networking instead - // pub network: Arc>, + pub network: Arc>, pub post_states: HashMap, - pub known_votes: VariableList, - pub new_votes: VariableList, + pub known_votes: Vec, + pub new_votes: Vec, pub dependencies: HashMap>, pub genesis_hash: B256, - // TODO: Proper validator key handling from static config pub num_validators: u64, pub safe_target: B256, pub head: B256, } impl Staker { - pub fn new(validator_id: u64, genesis_block: Block, genesis_state: LeanState) -> Staker { + pub fn new(validator_id: u64, network: Arc>, genesis_block: Block, genesis_state: LeanState) -> Staker { let genesis_hash = genesis_block.compute_hash(); Staker { validator_id, - public_key: PublicKey {}, - time: 0, - known_votes: VariableList::empty(), - new_votes: VariableList::empty(), + network: network, + known_votes: Vec::new(), + new_votes: Vec::new(), dependencies: HashMap::new(), genesis_hash, num_validators: genesis_state.config.num_validators, @@ -81,11 +74,11 @@ impl Staker { for new_vote in &self.new_votes { if !known_votes.any(|known_vote| known_vote == *new_vote) { - self.known_votes.push(new_vote.clone()).expect("Failed to push new_vote to known_votes"); + self.known_votes.push(new_vote.clone()); } } - self.new_votes = VariableList::empty(); + self.new_votes = Vec::new(); self.recompute_head(); } @@ -97,7 +90,7 @@ impl Staker { // Called every second pub fn tick(&mut self) { - let time_in_slot = self.time % SLOT_DURATION; + let time_in_slot = self.network.lock().unwrap().time % SLOT_DURATION; // t=0: propose a block if time_in_slot == 0 { @@ -123,7 +116,7 @@ impl Staker { } fn get_current_slot(&self) -> u64 { - self.time / SLOT_DURATION + 2 + self.network.lock().unwrap().time / SLOT_DURATION + 2 } // Called when it's the staker's turn to propose a block @@ -253,7 +246,7 @@ impl Staker { for vote in &block.votes { if !known_votes.any(|known_vote| known_vote == *vote) { - self.known_votes.push(vote.clone()).expect("Failed to push vote to known_votes"); + self.known_votes.push(vote.clone()); } } @@ -295,7 +288,7 @@ impl Staker { if is_known_vote || is_new_vote { // Do nothing } else if self.chain.contains_key(&vote.data.head) { - self.new_votes.push(vote.data.clone()).expect("Failed to push vote to new_votes"); + self.new_votes.push(vote.data.clone()); } else { self.dependencies .entry(vote.data.head) diff --git a/crates/networking/manager/Cargo.toml b/crates/networking/manager/Cargo.toml index c4e4f6a2c..deadc0d8d 100644 --- a/crates/networking/manager/Cargo.toml +++ b/crates/networking/manager/Cargo.toml @@ -34,6 +34,7 @@ ream-discv5.workspace = true ream-execution-engine.workspace = true ream-executor.workspace = true ream-fork-choice.workspace = true +ream-lean-chain.workspace = true ream-network-spec.workspace = true ream-operation-pool.workspace = true ream-p2p.workspace = true diff --git a/crates/networking/p2p/Cargo.toml b/crates/networking/p2p/Cargo.toml index c436c8f4a..59cdd9493 100644 --- a/crates/networking/p2p/Cargo.toml +++ b/crates/networking/p2p/Cargo.toml @@ -39,6 +39,7 @@ unsigned-varint = { version = "0.8", features = ["codec"] } # ream dependencies ream-consensus-beacon.workspace = true +ream-consensus-lean.workspace = true ream-consensus-misc.workspace = true ream-discv5.workspace = true ream-executor.workspace = true diff --git a/crates/networking/p2p/src/network/lean.rs b/crates/networking/p2p/src/network/lean.rs index 29837802f..2c903f030 100644 --- a/crates/networking/p2p/src/network/lean.rs +++ b/crates/networking/p2p/src/network/lean.rs @@ -1,10 +1,14 @@ use tracing::info; -pub struct NetworkService {} +pub struct NetworkService { + pub time: u64, +} impl NetworkService { pub async fn new() -> Self { - NetworkService {} + NetworkService { + time: 0, + } } pub async fn start(self) { From 183456edf76679d71c68371f4e482373e343bd10 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:03:27 +0700 Subject: [PATCH 22/47] fix: tracing::info! instead of println! --- crates/common/lean_chain/src/staker.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index c80e58ce5..e37e7ee3c 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -1,9 +1,4 @@ use alloy_primitives::B256; -use ream_p2p::network::lean::NetworkService; -use ream_pqc::PQSignature; -use ssz_types::VariableList; -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; use ream_consensus_lean::{ QueueItem, SLOT_DURATION, block::Block, @@ -11,6 +6,12 @@ use ream_consensus_lean::{ state::LeanState, vote::{SignedVote, Vote}, }; +use ream_p2p::network::lean::NetworkService; +use ream_pqc::PQSignature; +use ssz_types::VariableList; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use tracing::info; pub struct Staker { pub validator_id: u64, @@ -123,7 +124,7 @@ impl Staker { fn propose_block(&mut self) { let new_slot = self.get_current_slot(); - println!( + info!( "proposing (Staker {}), head = {}", self.validator_id, self.chain.get(&self.head).unwrap().slot @@ -208,7 +209,7 @@ impl Staker { signature: PQSignature {}, }; - println!( + info!( "voting (Staker {}), head = {}, t = {}, s = {}", self.validator_id, &self.chain.get(&self.head).unwrap().slot, From 3117b6f0589f93ef1c00e205d05e3ea66a15d1ba Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:05:37 +0700 Subject: [PATCH 23/47] chore: doc comments --- crates/common/consensus/lean/src/lib.rs | 14 +++++++------- crates/common/lean_chain/src/staker.rs | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index e6a3f1cdf..fbb909ce8 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -17,9 +17,9 @@ pub enum QueueItem { VoteItem(SignedVote), } -// We allow justification of slots either <= 5 or a perfect square or oblong after -// the latest finalized slot. This gives us a backoff technique and ensures -// finality keeps progressing even under high latency +/// We allow justification of slots either <= 5 or a perfect square or oblong after +/// the latest finalized slot. This gives us a backoff technique and ensures +/// finality keeps progressing even under high latency pub fn is_justifiable_slot(finalized_slot: &u64, candidate_slot: &u64) -> bool { assert!( candidate_slot >= finalized_slot, @@ -33,7 +33,7 @@ pub fn is_justifiable_slot(finalized_slot: &u64, candidate_slot: &u64) -> bool { || (delta as f64 + 0.25).sqrt() % 1.0 == 0.5 // any x^2+x } -// Given a state, output the new state after processing that block +/// Given a state, output the new state after processing that block pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { let mut state = pre_state.clone(); @@ -99,7 +99,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { state } -// Get the highest-slot justified block that we know about +/// Get the highest-slot justified block that we know about pub fn get_latest_justified_hash(post_states: &HashMap) -> Option { post_states .values() @@ -107,8 +107,8 @@ pub fn get_latest_justified_hash(post_states: &HashMap) -> Opti .map(|state| state.latest_justified_hash) } -// Use LMD GHOST to get the head, given a particular root (usually the -// latest known justified block) +/// Use LMD GHOST to get the head, given a particular root (usually the +/// latest known justified block) pub fn get_fork_choice_head( blocks: &HashMap, provided_root: &B256, diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index e37e7ee3c..d98524a4b 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -83,13 +83,13 @@ impl Staker { self.recompute_head(); } - // Done upon processing new votes or a new block + /// Done upon processing new votes or a new block fn recompute_head(&mut self) { let justified_hash = get_latest_justified_hash(&self.post_states).expect("Failed to get latest_justified_hash from post_states"); self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0); } - // Called every second + /// Called every second pub fn tick(&mut self) { let time_in_slot = self.network.lock().unwrap().time % SLOT_DURATION; @@ -120,7 +120,7 @@ impl Staker { self.network.lock().unwrap().time / SLOT_DURATION + 2 } - // Called when it's the staker's turn to propose a block + /// Called when it's the staker's turn to propose a block fn propose_block(&mut self) { let new_slot = self.get_current_slot(); @@ -174,7 +174,7 @@ impl Staker { // .submit(QueueItem::BlockItem(new_block), self.validator_id); } - // Called when it's the staker's turn to vote + /// Called when it's the staker's turn to vote fn vote(&mut self) { let state = self.post_states.get(&self.head).unwrap(); let mut target_block = self.chain.get(&self.head).unwrap(); @@ -225,7 +225,7 @@ impl Staker { // .submit(QueueItem::VoteItem(vote), self.validator_id); } - // Called by the p2p network + /// Called by the p2p network fn receive(&mut self, queue_item: &QueueItem) { match queue_item { QueueItem::BlockItem(block) => { From e0877d0e2f2800e28ad6ad5bb986def37e7ffa72 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:08:09 +0700 Subject: [PATCH 24/47] fix: make pr suggestions --- crates/common/consensus/lean/src/lib.rs | 45 ++++++++++++++++------- crates/common/consensus/lean/src/state.rs | 5 ++- crates/common/lean_chain/src/staker.rs | 24 +++++++++--- crates/networking/p2p/src/network/lean.rs | 4 +- 4 files changed, 53 insertions(+), 25 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index fbb909ce8..de2f8211e 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -3,11 +3,16 @@ pub mod config; pub mod state; pub mod vote; +use std::collections::HashMap; + use alloy_primitives::B256; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use crate::{block::Block, state::LeanState, vote::SignedVote, vote::Vote}; +use crate::{ + block::Block, + state::LeanState, + vote::{SignedVote, Vote}, +}; pub const SLOT_DURATION: u64 = 12; @@ -38,12 +43,24 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { let mut state = pre_state.clone(); // Track historical blocks in the state - state.historical_block_hashes.push(block.parent).expect("Failed to add block.parent to historical_block_hashes"); - state.justified_slots.push(false).expect("Failed to add to justified_slots"); + state + .historical_block_hashes + .push(block.parent) + .expect("Failed to add block.parent to historical_block_hashes"); + state + .justified_slots + .push(false) + .expect("Failed to add to justified_slots"); while state.historical_block_hashes.len() < block.slot as usize { - state.justified_slots.push(false).expect("Failed to prefill justified_slots"); - state.historical_block_hashes.push(None).expect("Failed to prefill historical_block_hashes"); + state + .justified_slots + .push(false) + .expect("Failed to prefill justified_slots"); + state + .historical_block_hashes + .push(None) + .expect("Failed to prefill historical_block_hashes"); } // Process votes @@ -62,9 +79,10 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Track attempts to justify new hashes if !state.justifications.contains_key(&vote.target) { - state - .justifications - .insert(vote.target, vec![false; state.config.num_validators as usize]); + state.justifications.insert( + vote.target, + vec![false; state.config.num_validators as usize], + ); } if !state.justifications[&vote.target][vote.validator_id as usize] { @@ -85,8 +103,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Finalization: if the target is the next valid justifiable // hash after the source - let is_target_next_valid_justifiable_slot = !((vote.source_slot + 1) - ..vote.target_slot) + let is_target_next_valid_justifiable_slot = !((vote.source_slot + 1)..vote.target_slot) .any(|slot| is_justifiable_slot(&state.latest_finalized_slot, &slot)); if is_target_next_valid_justifiable_slot { @@ -112,7 +129,7 @@ pub fn get_latest_justified_hash(post_states: &HashMap) -> Opti pub fn get_fork_choice_head( blocks: &HashMap, provided_root: &B256, - votes: &Vec, + votes: &[Vote], min_score: u64, ) -> B256 { let mut root = *provided_root; @@ -129,7 +146,7 @@ pub fn get_fork_choice_head( // Identify latest votes // Sort votes by ascending slots to ensure that new votes are inserted last - let mut sorted_votes = votes.clone(); + let mut sorted_votes = votes.to_owned(); sorted_votes.sort_by_key(|vote| vote.slot); // Prepare a map of validator_id -> their vote @@ -162,7 +179,7 @@ pub fn get_fork_choice_head( if block.parent.is_some() && *vote_weights.get(hash).unwrap_or(&0) >= min_score { children_map .entry(block.parent.unwrap()) - .or_insert_with(Vec::new) + .or_default() .push(*hash); } } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 07913e9cb..0eb3e7838 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,8 +1,9 @@ +use std::collections::HashMap; + use alloy_primitives::B256; use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; -use ssz_types::{typenum::U4096, VariableList}; -use std::collections::HashMap; +use ssz_types::{VariableList, typenum::U4096}; use crate::config::Config; diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index d98524a4b..d797f9b9e 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -1,3 +1,8 @@ +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + use alloy_primitives::B256; use ream_consensus_lean::{ QueueItem, SLOT_DURATION, @@ -9,8 +14,6 @@ use ream_consensus_lean::{ use ream_p2p::network::lean::NetworkService; use ream_pqc::PQSignature; use ssz_types::VariableList; -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; use tracing::info; pub struct Staker { @@ -28,12 +31,17 @@ pub struct Staker { } impl Staker { - pub fn new(validator_id: u64, network: Arc>, genesis_block: Block, genesis_state: LeanState) -> Staker { + pub fn new( + validator_id: u64, + network: Arc>, + genesis_block: Block, + genesis_state: LeanState, + ) -> Staker { let genesis_hash = genesis_block.compute_hash(); Staker { validator_id, - network: network, + network, known_votes: Vec::new(), new_votes: Vec::new(), dependencies: HashMap::new(), @@ -85,7 +93,8 @@ impl Staker { /// Done upon processing new votes or a new block fn recompute_head(&mut self) { - let justified_hash = get_latest_justified_hash(&self.post_states).expect("Failed to get latest_justified_hash from post_states"); + let justified_hash = get_latest_justified_hash(&self.post_states) + .expect("Failed to get latest_justified_hash from post_states"); self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0); } @@ -158,7 +167,10 @@ impl Staker { for vote in new_votes_to_add { // TODO: proper error handling - new_block.votes.push(vote).expect("Failed to add vote to new_block"); + new_block + .votes + .push(vote) + .expect("Failed to add vote to new_block"); } } diff --git a/crates/networking/p2p/src/network/lean.rs b/crates/networking/p2p/src/network/lean.rs index 2c903f030..d48dd93c0 100644 --- a/crates/networking/p2p/src/network/lean.rs +++ b/crates/networking/p2p/src/network/lean.rs @@ -6,9 +6,7 @@ pub struct NetworkService { impl NetworkService { pub async fn new() -> Self { - NetworkService { - time: 0, - } + NetworkService { time: 0 } } pub async fn start(self) { From 4a3ccca0db0003d2e356e89bbaeacb85ba5707a9 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:28:14 +0700 Subject: [PATCH 25/47] feat: replace compute_hash() with TreeHash --- Cargo.lock | 1 + crates/common/consensus/lean/src/block.rs | 28 ++++++----------------- crates/common/consensus/lean/src/lib.rs | 10 ++++---- crates/common/lean_chain/Cargo.toml | 1 + crates/common/lean_chain/src/staker.rs | 23 ++++++++++--------- 5 files changed, 27 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9410b8a0d..22716660a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5456,6 +5456,7 @@ dependencies = [ "ssz_types", "tokio", "tracing", + "tree_hash", ] [[package]] diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 5dd71ddc1..386738ecc 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -1,36 +1,22 @@ use alloy_primitives::B256; -use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; -use ssz_types::{ - VariableList, - typenum::{ - U16777216, // 2**24 - }, -}; +use ssz_types::{VariableList, typenum::U4096}; +use tree_hash_derive::TreeHash; use crate::vote::Vote; -// TODO: Add back #[derive(TreeHash)] -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct SignedBlock { pub message: Block, pub signature: PQSignature, } -// TODO: Add back #[derive(TreeHash)] -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, Default)] +#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct Block { pub slot: u64, - pub parent: Option, - pub votes: VariableList, - pub state_root: Option, -} - -impl Block { - pub fn compute_hash(&self) -> B256 { - let serialized = serde_json::to_string(self).unwrap(); - B256::from_slice(&hash(serialized.as_bytes())) - } + pub parent: B256, + pub votes: VariableList, + pub state_root: B256, } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index de2f8211e..21c139e4c 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -45,7 +45,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Track historical blocks in the state state .historical_block_hashes - .push(block.parent) + .push(Some(block.parent)) .expect("Failed to add block.parent to historical_block_hashes"); state .justified_slots @@ -167,7 +167,7 @@ pub fn get_fork_choice_head( while blocks.get(&block_hash).unwrap().slot > blocks.get(&root).unwrap().slot { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); vote_weights.insert(block_hash, current_weights + 1); - block_hash = blocks.get(&block_hash).unwrap().parent.unwrap(); + block_hash = blocks.get(&block_hash).unwrap().parent; } } } @@ -176,9 +176,11 @@ pub fn get_fork_choice_head( let mut children_map = HashMap::>::new(); for (hash, block) in blocks { - if block.parent.is_some() && *vote_weights.get(hash).unwrap_or(&0) >= min_score { + // Original Python impl uses `block.parent` to imply that the block has a parent, + // So for Rust, we use `block.parent != B256::ZERO` instead. + if block.parent != B256::ZERO && *vote_weights.get(hash).unwrap_or(&0) >= min_score { children_map - .entry(block.parent.unwrap()) + .entry(block.parent) .or_default() .push(*hash); } diff --git a/crates/common/lean_chain/Cargo.toml b/crates/common/lean_chain/Cargo.toml index d29715a1b..124855407 100644 --- a/crates/common/lean_chain/Cargo.toml +++ b/crates/common/lean_chain/Cargo.toml @@ -15,6 +15,7 @@ serde.workspace = true ssz_types.workspace = true tokio.workspace = true tracing.workspace = true +tree_hash.workspace = true # ream dependencies ream-consensus-lean.workspace = true diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index d797f9b9e..29cb31564 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -15,6 +15,7 @@ use ream_p2p::network::lean::NetworkService; use ream_pqc::PQSignature; use ssz_types::VariableList; use tracing::info; +use tree_hash::TreeHash; pub struct Staker { pub validator_id: u64, @@ -37,7 +38,7 @@ impl Staker { genesis_block: Block, genesis_state: LeanState, ) -> Staker { - let genesis_hash = genesis_block.compute_hash(); + let genesis_hash = genesis_block.tree_hash_root(); Staker { validator_id, @@ -142,9 +143,9 @@ impl Staker { let head_state = self.post_states.get(&self.head).unwrap(); let mut new_block = Block { slot: new_slot, - parent: Some(self.head), + parent: self.head, votes: VariableList::empty(), - state_root: None, + state_root: B256::ZERO, }; let mut state: LeanState; @@ -174,8 +175,8 @@ impl Staker { } } - new_block.state_root = Some(state.compute_hash()); - let new_hash = new_block.compute_hash(); + new_block.state_root = state.compute_hash(); + let new_hash = new_block.tree_hash_root(); self.chain.insert(new_hash, new_block.clone()); self.post_states.insert(new_hash, state); @@ -195,14 +196,14 @@ impl Staker { // of the head for _ in 0..3 { if target_block.slot > self.chain.get(&self.safe_target).unwrap().slot { - target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); + target_block = self.chain.get(&target_block.parent).unwrap(); } } // If the latest finalized slot is very far back, then only some slots are // valid to justify, make sure the target is one of those while !is_justifiable_slot(&state.latest_finalized_slot, &target_block.slot) { - target_block = self.chain.get(&target_block.parent.unwrap()).unwrap(); + target_block = self.chain.get(&target_block.parent).unwrap(); } let vote = Vote { @@ -210,7 +211,7 @@ impl Staker { slot: self.get_current_slot(), head: self.head, head_slot: self.chain.get(&self.head).unwrap().slot, - target: target_block.compute_hash(), + target: target_block.tree_hash_root(), target_slot: target_block.slot, source: state.latest_justified_hash, source_slot: state.latest_justified_slot, @@ -241,14 +242,14 @@ impl Staker { fn receive(&mut self, queue_item: &QueueItem) { match queue_item { QueueItem::BlockItem(block) => { - let block_hash = block.compute_hash(); + let block_hash = block.tree_hash_root(); // If the block is already known, ignore it if self.chain.contains_key(&block_hash) { return; } - match self.post_states.get(&block.parent.unwrap()) { + match self.post_states.get(&block.parent) { Some(parent_state) => { let state = process_block(parent_state, block); @@ -279,7 +280,7 @@ impl Staker { // If we have not yet seen the block's parent, ignore for now, // process later once we actually see the parent self.dependencies - .entry(block.parent.unwrap()) + .entry(block.parent) .or_default() .push(queue_item.clone()); } From 172bcc1f8b37b8dff17bebb96cd556ec0fc6aac2 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 15:43:52 +0700 Subject: [PATCH 26/47] fix: make pr --- crates/common/consensus/lean/src/block.rs | 4 +++- crates/common/consensus/lean/src/lib.rs | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 386738ecc..7da7daa65 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -13,7 +13,9 @@ pub struct SignedBlock { pub signature: PQSignature, } -#[derive(Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] +#[derive( + Debug, Default, PartialEq, Eq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, +)] pub struct Block { pub slot: u64, pub parent: B256, diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 21c139e4c..758f02e16 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -179,10 +179,7 @@ pub fn get_fork_choice_head( // Original Python impl uses `block.parent` to imply that the block has a parent, // So for Rust, we use `block.parent != B256::ZERO` instead. if block.parent != B256::ZERO && *vote_weights.get(hash).unwrap_or(&0) >= min_score { - children_map - .entry(block.parent) - .or_default() - .push(*hash); + children_map.entry(block.parent).or_default().push(*hash); } } From 70254cc982e30eb40ded89c08cea79bd5ba51a5d Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:26:22 +0700 Subject: [PATCH 27/47] feat: implement flat justifications --- Cargo.lock | 14 +++--- Cargo.toml | 2 +- crates/common/consensus/lean/src/lib.rs | 20 +++------ crates/common/consensus/lean/src/state.rs | 52 +++++++++++++++++++++-- crates/networking/p2p/Cargo.toml | 1 - 5 files changed, 60 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22716660a..3fdf6c562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2133,7 +2133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4983,7 +4983,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5568,7 +5568,6 @@ dependencies = [ "libp2p-mplex", "parking_lot", "ream-consensus-beacon", - "ream-consensus-lean", "ream-consensus-misc", "ream-discv5", "ream-executor", @@ -6097,7 +6096,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6110,7 +6109,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6666,8 +6665,7 @@ dependencies = [ [[package]] name = "ssz_types" version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b55bedc9a18ed2860a46d6beb4f4082416ee1d60be0cc364cebdcdddc7afd4" +source = "git+https://github.com/ReamLabs/ssz_types?branch=removable-variable-list#01de4ea4385064ca3985bfc3ab24e825f107c64c" dependencies = [ "ethereum_serde_utils", "ethereum_ssz", @@ -6832,7 +6830,7 @@ dependencies = [ "getrandom 0.3.2", "once_cell", "rustix 1.0.5", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb2471680..422403807 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,7 +100,7 @@ serde_json = "1.0.139" serde_yaml = "0.9" sha2 = "0.10" snap = "1.1" -ssz_types = "0.11" +ssz_types = { git = "https://github.com/ReamLabs/ssz_types", branch = "removable-variable-list" } tempdir = "0.3.7" tempfile = "3.19" thiserror = "2.0.11" diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 758f02e16..885207464 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -78,28 +78,18 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { } // Track attempts to justify new hashes - if !state.justifications.contains_key(&vote.target) { - state.justifications.insert( - vote.target, - vec![false; state.config.num_validators as usize], - ); - } - - if !state.justifications[&vote.target][vote.validator_id as usize] { - state.justifications.get_mut(&vote.target).unwrap()[vote.validator_id as usize] = true; - } + state.initialize_justifications_for_root(&vote.target); + state.set_justification(&vote.target, &vote.validator_id, true); - let count = state.justifications[&vote.target] - .iter() - .fold(0, |sum, justification| sum + *justification as usize); + let count = state.count_justifications(&vote.target); // If 2/3 voted for the same new valid hash to justify - if count == (2 * state.config.num_validators as usize) / 3 { + if count == (2 * state.config.num_validators) / 3 { state.latest_justified_hash = vote.target; state.latest_justified_slot = vote.target_slot; state.justified_slots[vote.target_slot as usize] = true; - state.justifications.remove(&vote.target).unwrap(); + state.remove_justifications(&vote.target); // Finalization: if the target is the next valid justifiable // hash after the source diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 0eb3e7838..6899e987b 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,9 +1,7 @@ -use std::collections::HashMap; - use alloy_primitives::B256; use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; -use ssz_types::{VariableList, typenum::U4096}; +use ssz_types::{BitList, VariableList, typenum::{Unsigned, U4096, U16777216}}; use crate::config::Config; @@ -20,7 +18,9 @@ pub struct LeanState { pub historical_block_hashes: VariableList, U4096>, pub justified_slots: VariableList, - pub justifications: HashMap>, + // Originally `justifications: Dict[str, List[bool]]` + pub justifications_roots: VariableList, + pub justifications_roots_validators: BitList, } impl LeanState { @@ -28,4 +28,48 @@ impl LeanState { let serialized = serde_json::to_string(self).unwrap(); B256::from_slice(&hash(serialized.as_bytes())) } + + fn get_justifications_roots_index(&self, root: &B256) -> Option { + self.justifications_roots.iter().position(|r| r == root) + } + + pub fn initialize_justifications_for_root(&mut self, root: &B256) { + if !self.justifications_roots.contains(root) { + self.justifications_roots.push(root.clone()).expect("Failed to insert root into justifications_roots"); + } + } + + pub fn set_justification(&mut self, root: &B256, validator_id: &u64, value: bool) { + let index = self.get_justifications_roots_index(root).expect("Failed to find the justifications index to set"); + self.justifications_roots_validators.set(index * U4096::to_usize() + *validator_id as usize, value).expect("Failed to set justification bit"); + } + + pub fn count_justifications(&self, root: &B256) -> u64 { + let index = self + .get_justifications_roots_index(root) + .expect("Could not find justifications for the provided block root"); + + let start_range = index * U4096::to_usize(); + let end_range = start_range + U4096::to_usize(); + + self + .justifications_roots_validators + .as_slice()[start_range..end_range] + .iter() + .fold(0, |acc, justification_bits| acc + justification_bits.count_ones()) as u64 + } + + pub fn remove_justifications(&mut self, root: &B256) { + // Remove from `state.justifications_roots` + let index = self.get_justifications_roots_index(root).expect("Failed to find the justifications index to remove"); + self.justifications_roots.remove(index); + + let start_range = index * U4096::to_usize(); + let end_range = start_range + U4096::to_usize(); + + // Remove from `state.justifications_roots_validators` + for i in start_range..end_range { + self.justifications_roots_validators.set(i, false).expect("Failed to remove justifications"); + } + } } diff --git a/crates/networking/p2p/Cargo.toml b/crates/networking/p2p/Cargo.toml index 59cdd9493..c436c8f4a 100644 --- a/crates/networking/p2p/Cargo.toml +++ b/crates/networking/p2p/Cargo.toml @@ -39,7 +39,6 @@ unsigned-varint = { version = "0.8", features = ["codec"] } # ream dependencies ream-consensus-beacon.workspace = true -ream-consensus-lean.workspace = true ream-consensus-misc.workspace = true ream-discv5.workspace = true ream-executor.workspace = true From 4296b13b32ec224fb06c3c0b0a82b7806920b301 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:27:07 +0700 Subject: [PATCH 28/47] fix: formatting --- crates/common/consensus/lean/src/state.rs | 33 ++++++++++++++++------- crates/common/consensus/lean/src/vote.rs | 8 ------ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 6899e987b..ec70ce2d1 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,7 +1,10 @@ use alloy_primitives::B256; use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; -use ssz_types::{BitList, VariableList, typenum::{Unsigned, U4096, U16777216}}; +use ssz_types::{ + BitList, VariableList, + typenum::{U4096, U16777216, Unsigned}, +}; use crate::config::Config; @@ -35,13 +38,19 @@ impl LeanState { pub fn initialize_justifications_for_root(&mut self, root: &B256) { if !self.justifications_roots.contains(root) { - self.justifications_roots.push(root.clone()).expect("Failed to insert root into justifications_roots"); + self.justifications_roots + .push(*root) + .expect("Failed to insert root into justifications_roots"); } } pub fn set_justification(&mut self, root: &B256, validator_id: &u64, value: bool) { - let index = self.get_justifications_roots_index(root).expect("Failed to find the justifications index to set"); - self.justifications_roots_validators.set(index * U4096::to_usize() + *validator_id as usize, value).expect("Failed to set justification bit"); + let index = self + .get_justifications_roots_index(root) + .expect("Failed to find the justifications index to set"); + self.justifications_roots_validators + .set(index * U4096::to_usize() + *validator_id as usize, value) + .expect("Failed to set justification bit"); } pub fn count_justifications(&self, root: &B256) -> u64 { @@ -52,16 +61,18 @@ impl LeanState { let start_range = index * U4096::to_usize(); let end_range = start_range + U4096::to_usize(); - self - .justifications_roots_validators - .as_slice()[start_range..end_range] + self.justifications_roots_validators.as_slice()[start_range..end_range] .iter() - .fold(0, |acc, justification_bits| acc + justification_bits.count_ones()) as u64 + .fold(0, |acc, justification_bits| { + acc + justification_bits.count_ones() + }) as u64 } pub fn remove_justifications(&mut self, root: &B256) { // Remove from `state.justifications_roots` - let index = self.get_justifications_roots_index(root).expect("Failed to find the justifications index to remove"); + let index = self + .get_justifications_roots_index(root) + .expect("Failed to find the justifications index to remove"); self.justifications_roots.remove(index); let start_range = index * U4096::to_usize(); @@ -69,7 +80,9 @@ impl LeanState { // Remove from `state.justifications_roots_validators` for i in start_range..end_range { - self.justifications_roots_validators.set(i, false).expect("Failed to remove justifications"); + self.justifications_roots_validators + .set(i, false) + .expect("Failed to remove justifications"); } } } diff --git a/crates/common/consensus/lean/src/vote.rs b/crates/common/consensus/lean/src/vote.rs index ac3be2a01..1169b5076 100644 --- a/crates/common/consensus/lean/src/vote.rs +++ b/crates/common/consensus/lean/src/vote.rs @@ -1,5 +1,4 @@ use alloy_primitives::B256; -use ethereum_hashing::hash; use ream_pqc::PQSignature; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; @@ -22,10 +21,3 @@ pub struct Vote { pub source: B256, pub source_slot: u64, } - -impl Vote { - pub fn compute_hash(&self) -> B256 { - let serialized = serde_json::to_string(self).unwrap(); - B256::from_slice(&hash(serialized.as_bytes())) - } -} From 0cf3728c696b37a929742a38e6dc78914310c08c Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:33:06 +0700 Subject: [PATCH 29/47] feat: replace compute_hash() with tree_hash_root() for LeanState --- crates/common/consensus/lean/src/lib.rs | 10 ++++++---- crates/common/consensus/lean/src/state.rs | 13 ++++--------- crates/common/lean_chain/src/staker.rs | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 885207464..e9aeee886 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -45,7 +45,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // Track historical blocks in the state state .historical_block_hashes - .push(Some(block.parent)) + .push(block.parent) .expect("Failed to add block.parent to historical_block_hashes"); state .justified_slots @@ -57,9 +57,11 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { .justified_slots .push(false) .expect("Failed to prefill justified_slots"); + state .historical_block_hashes - .push(None) + // Divergent from Python ref implementation: uses `B256::ZERO` instead of `None` + .push(B256::ZERO) .expect("Failed to prefill historical_block_hashes"); } @@ -69,8 +71,8 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { // or whose target is not in the history, or whose target is not a // valid justifiable slot if !state.justified_slots[vote.source_slot as usize] - || Some(vote.source) != state.historical_block_hashes[vote.source_slot as usize] - || Some(vote.target) != state.historical_block_hashes[vote.target_slot as usize] + || vote.source != state.historical_block_hashes[vote.source_slot as usize] + || vote.target != state.historical_block_hashes[vote.target_slot as usize] || vote.target_slot <= vote.source_slot || !is_justifiable_slot(&state.latest_finalized_slot, &vote.target_slot) { diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index ec70ce2d1..54f9b2532 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,15 +1,15 @@ use alloy_primitives::B256; -use ethereum_hashing::hash; use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; use ssz_types::{ BitList, VariableList, typenum::{U4096, U16777216, Unsigned}, }; +use tree_hash_derive::TreeHash; use crate::config::Config; -// TODO: Add back #[derive(Encode, Decode, TreeHash)] -#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] +#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct LeanState { pub config: Config, @@ -18,7 +18,7 @@ pub struct LeanState { pub latest_finalized_hash: B256, pub latest_finalized_slot: u64, - pub historical_block_hashes: VariableList, U4096>, + pub historical_block_hashes: VariableList, pub justified_slots: VariableList, // Originally `justifications: Dict[str, List[bool]]` @@ -27,11 +27,6 @@ pub struct LeanState { } impl LeanState { - pub fn compute_hash(&self) -> B256 { - let serialized = serde_json::to_string(self).unwrap(); - B256::from_slice(&hash(serialized.as_bytes())) - } - fn get_justifications_roots_index(&self, root: &B256) -> Option { self.justifications_roots.iter().position(|r| r == root) } diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index 29cb31564..686c5ea37 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -175,7 +175,7 @@ impl Staker { } } - new_block.state_root = state.compute_hash(); + new_block.state_root = state.tree_hash_root(); let new_hash = new_block.tree_hash_root(); self.chain.insert(new_hash, new_block.clone()); From 534f70548b145d17848fe786294c5fcb7097046e Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:43:20 +0700 Subject: [PATCH 30/47] docs: add comments from Python impl --- crates/common/consensus/lean/src/block.rs | 2 ++ crates/common/consensus/lean/src/lib.rs | 2 +- crates/common/consensus/lean/src/state.rs | 1 + crates/common/lean_chain/src/staker.rs | 16 ++++++++++++++-- 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/common/consensus/lean/src/block.rs b/crates/common/consensus/lean/src/block.rs index 7da7daa65..0c55eb0f0 100644 --- a/crates/common/consensus/lean/src/block.rs +++ b/crates/common/consensus/lean/src/block.rs @@ -18,7 +18,9 @@ pub struct SignedBlock { )] pub struct Block { pub slot: u64, + // Diverged from Python implementation: Disallow `None` (uses `B256::ZERO` instead) pub parent: B256, pub votes: VariableList, + // Diverged from Python implementation: Disallow `None` (uses `B256::ZERO` instead) pub state_root: B256, } diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index e9aeee886..53dc01af1 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -60,7 +60,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { state .historical_block_hashes - // Divergent from Python ref implementation: uses `B256::ZERO` instead of `None` + // Diverged from Python implementation: uses `B256::ZERO` instead of `None` .push(B256::ZERO) .expect("Failed to prefill historical_block_hashes"); } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 54f9b2532..4e13d35cb 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -21,6 +21,7 @@ pub struct LeanState { pub historical_block_hashes: VariableList, pub justified_slots: VariableList, + // Diverged from Python implementation: // Originally `justifications: Dict[str, List[bool]]` pub justifications_roots: VariableList, pub justifications_roots_validators: BitList, diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index 686c5ea37..f35cc61c3 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -41,17 +41,28 @@ impl Staker { let genesis_hash = genesis_block.tree_hash_root(); Staker { + // This node's validator ID validator_id, + // Hook to the p2p network network, + // {block_hash: block} for all blocks that we know about + chain: HashMap::from([(genesis_hash, genesis_block)]), + // {block_hash: post_state} for all blocks that we know about + post_states: HashMap::from([(genesis_hash, genesis_state)]), + // Votes that we have received and taken into account known_votes: Vec::new(), + // Votes that we have received but not yet taken into account new_votes: Vec::new(), + // Objects that we will process once we have processed their parents dependencies: HashMap::new(), + // Initialize the chain with the genesis block genesis_hash, num_validators: genesis_state.config.num_validators, + // Block that it is safe to use to vote as the target + // Diverge from Python implementation: Use genesis hash instead of `None` safe_target: genesis_hash, + // Head of the chain head: genesis_hash, - chain: HashMap::from([(genesis_hash, genesis_block)]), - post_states: HashMap::from([(genesis_hash, genesis_state)]), } } @@ -145,6 +156,7 @@ impl Staker { slot: new_slot, parent: self.head, votes: VariableList::empty(), + // Diverged from Python implementation: Using `B256::ZERO` instead of `None`) state_root: B256::ZERO, }; let mut state: LeanState; From 7a74b7b7828691878e9e84445b06109782d97509 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:48:39 +0700 Subject: [PATCH 31/47] fix: remove unused deps --- crates/networking/manager/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/networking/manager/Cargo.toml b/crates/networking/manager/Cargo.toml index deadc0d8d..c4e4f6a2c 100644 --- a/crates/networking/manager/Cargo.toml +++ b/crates/networking/manager/Cargo.toml @@ -34,7 +34,6 @@ ream-discv5.workspace = true ream-execution-engine.workspace = true ream-executor.workspace = true ream-fork-choice.workspace = true -ream-lean-chain.workspace = true ream-network-spec.workspace = true ream-operation-pool.workspace = true ream-p2p.workspace = true From 0da5620f25029583363f25271c6cd2bf13aa23a0 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 1 Aug 2025 17:54:16 +0700 Subject: [PATCH 32/47] fix: Staker sequence --- Cargo.lock | 1 - crates/common/lean_chain/src/staker.rs | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3fdf6c562..c874f5d8b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5504,7 +5504,6 @@ dependencies = [ "ream-execution-engine", "ream-executor", "ream-fork-choice", - "ream-lean-chain", "ream-network-spec", "ream-operation-pool", "ream-p2p", diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index f35cc61c3..0a45ae9ef 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -45,10 +45,6 @@ impl Staker { validator_id, // Hook to the p2p network network, - // {block_hash: block} for all blocks that we know about - chain: HashMap::from([(genesis_hash, genesis_block)]), - // {block_hash: post_state} for all blocks that we know about - post_states: HashMap::from([(genesis_hash, genesis_state)]), // Votes that we have received and taken into account known_votes: Vec::new(), // Votes that we have received but not yet taken into account @@ -63,6 +59,10 @@ impl Staker { safe_target: genesis_hash, // Head of the chain head: genesis_hash, + // {block_hash: block} for all blocks that we know about + chain: HashMap::from([(genesis_hash, genesis_block)]), + // {block_hash: post_state} for all blocks that we know about + post_states: HashMap::from([(genesis_hash, genesis_state)]), } } From 87ea21f563dc1dee31b50c153902014ce9a5e7b1 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 12:06:24 +0700 Subject: [PATCH 33/47] feat: update max slots from 4096 to 262144 --- crates/common/consensus/lean/src/state.rs | 24 ++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 4e13d35cb..937b1a3dc 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{ BitList, VariableList, - typenum::{U4096, U16777216, Unsigned}, + typenum::{U4096, U262144, U16777216, Unsigned}, }; use tree_hash_derive::TreeHash; @@ -18,12 +18,12 @@ pub struct LeanState { pub latest_finalized_hash: B256, pub latest_finalized_slot: u64, - pub historical_block_hashes: VariableList, - pub justified_slots: VariableList, + pub historical_block_hashes: VariableList, + pub justified_slots: VariableList, // Diverged from Python implementation: // Originally `justifications: Dict[str, List[bool]]` - pub justifications_roots: VariableList, + pub justifications_roots: VariableList, pub justifications_roots_validators: BitList, } @@ -32,6 +32,14 @@ impl LeanState { self.justifications_roots.iter().position(|r| r == root) } + fn get_justifications_roots_range(&self, index: &usize) -> (usize, usize) { + // Start from index * MAX_HISTORICAL_BLOCK_HASHES, ends at start + VALIDATOR_REGISTRY_LIMIT + let start_range = index * U262144::to_usize(); + let end_range = start_range + U4096::to_usize(); + + (start_range, end_range) + } + pub fn initialize_justifications_for_root(&mut self, root: &B256) { if !self.justifications_roots.contains(root) { self.justifications_roots @@ -45,7 +53,7 @@ impl LeanState { .get_justifications_roots_index(root) .expect("Failed to find the justifications index to set"); self.justifications_roots_validators - .set(index * U4096::to_usize() + *validator_id as usize, value) + .set(index * U262144::to_usize() + *validator_id as usize, value) .expect("Failed to set justification bit"); } @@ -54,8 +62,7 @@ impl LeanState { .get_justifications_roots_index(root) .expect("Could not find justifications for the provided block root"); - let start_range = index * U4096::to_usize(); - let end_range = start_range + U4096::to_usize(); + let (start_range, end_range) = self.get_justifications_roots_range(&index); self.justifications_roots_validators.as_slice()[start_range..end_range] .iter() @@ -71,8 +78,7 @@ impl LeanState { .expect("Failed to find the justifications index to remove"); self.justifications_roots.remove(index); - let start_range = index * U4096::to_usize(); - let end_range = start_range + U4096::to_usize(); + let (start_range, end_range) = self.get_justifications_roots_range(&index); // Remove from `state.justifications_roots_validators` for i in start_range..end_range { From bb6458ae101c7976adb1fc9f69cc19ca988172e8 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 12:22:09 +0700 Subject: [PATCH 34/47] refactor: better vector filters --- crates/common/lean_chain/src/staker.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index 0a45ae9ef..eb841b2bb 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -91,10 +91,8 @@ impl Staker { /// Process new votes that the staker has received. Vote processing is done /// at a particular time, because of safe target and view merge rule fn accept_new_votes(&mut self) { - let mut known_votes = self.known_votes.clone().into_iter(); - for new_vote in &self.new_votes { - if !known_votes.any(|known_vote| known_vote == *new_vote) { + if !self.known_votes.contains(new_vote) { self.known_votes.push(new_vote.clone()); } } @@ -165,14 +163,13 @@ impl Staker { loop { state = process_block(head_state, &new_block); - let mut new_votes_to_add = Vec::::new(); - for vote in self.known_votes.clone().into_iter() { - if vote.source == state.latest_justified_hash - && !new_block.votes.clone().into_iter().any(|v| v == vote) - { - new_votes_to_add.push(vote); - } - } + let new_votes_to_add = self + .known_votes + .clone() + .into_iter() + .filter(|vote| vote.source == state.latest_justified_hash) + .filter(|vote| !new_block.votes.contains(vote)) + .collect::>(); if new_votes_to_add.is_empty() { break; From 6a8c0d8cc24b2d17368d2208ca707057e00af98d Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 13:33:27 +0700 Subject: [PATCH 35/47] fix: Use U1073741824 for justifications and integers when typenum is not necessary --- crates/common/consensus/lean/src/lib.rs | 2 ++ crates/common/consensus/lean/src/state.rs | 17 +++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 53dc01af1..631943d1f 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -15,6 +15,8 @@ use crate::{ }; pub const SLOT_DURATION: u64 = 12; +pub const MAX_HISTORICAL_BLOCK_HASHES: u64 = 262144; +pub const VALIDATOR_REGISTRY_LIMIT: u64 = 4096; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] pub enum QueueItem { diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 937b1a3dc..5935bdd95 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -3,11 +3,15 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{ BitList, VariableList, - typenum::{U4096, U262144, U16777216, Unsigned}, + typenum::{U4096, U262144, U1073741824, Unsigned}, }; use tree_hash_derive::TreeHash; -use crate::config::Config; +use crate::{ + config::Config + MAX_HISTORICAL_BLOCK_HASHES, + VALIDATOR_REGISTRY_LIMIT, +}; #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct LeanState { @@ -24,7 +28,9 @@ pub struct LeanState { // Diverged from Python implementation: // Originally `justifications: Dict[str, List[bool]]` pub justifications_roots: VariableList, - pub justifications_roots_validators: BitList, + // The size is MAX_HISTORICAL_BLOCK_HASHES * VALIDATOR_REGISTRY_LIMIT + // to accommodate equivalent to `justifications[root][validator_id]` + pub justifications_roots_validators: BitList, } impl LeanState { @@ -33,9 +39,8 @@ impl LeanState { } fn get_justifications_roots_range(&self, index: &usize) -> (usize, usize) { - // Start from index * MAX_HISTORICAL_BLOCK_HASHES, ends at start + VALIDATOR_REGISTRY_LIMIT - let start_range = index * U262144::to_usize(); - let end_range = start_range + U4096::to_usize(); + let start_range = index * MAX_HISTORICAL_BLOCK_HASHES; + let end_range = start_range + VALIDATOR_REGISTRY_LIMIT; (start_range, end_range) } From 2e09fcc4975b0c126eddad51edaeb1055fae4703 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 14:26:00 +0700 Subject: [PATCH 36/47] fix: type comparison --- crates/common/consensus/lean/src/state.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 5935bdd95..3f04aa4be 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -3,12 +3,12 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{ BitList, VariableList, - typenum::{U4096, U262144, U1073741824, Unsigned}, + typenum::{U262144, U1073741824, Unsigned}, }; use tree_hash_derive::TreeHash; use crate::{ - config::Config + config::Config, MAX_HISTORICAL_BLOCK_HASHES, VALIDATOR_REGISTRY_LIMIT, }; @@ -39,8 +39,8 @@ impl LeanState { } fn get_justifications_roots_range(&self, index: &usize) -> (usize, usize) { - let start_range = index * MAX_HISTORICAL_BLOCK_HASHES; - let end_range = start_range + VALIDATOR_REGISTRY_LIMIT; + let start_range = index * MAX_HISTORICAL_BLOCK_HASHES as usize; + let end_range = start_range + VALIDATOR_REGISTRY_LIMIT as usize; (start_range, end_range) } From 789692cd8a2e29570d37481ed125050c47348ac6 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 14:26:10 +0700 Subject: [PATCH 37/47] fix: remove clones --- crates/common/lean_chain/src/staker.rs | 45 +++++++++----------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index eb841b2bb..8d6f7feb0 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -91,13 +91,12 @@ impl Staker { /// Process new votes that the staker has received. Vote processing is done /// at a particular time, because of safe target and view merge rule fn accept_new_votes(&mut self) { - for new_vote in &self.new_votes { - if !self.known_votes.contains(new_vote) { - self.known_votes.push(new_vote.clone()); + for new_vote in self.new_votes.drain(..) { + if !self.known_votes.contains(&new_vote) { + self.known_votes.push(new_vote); } } - self.new_votes = Vec::new(); self.recompute_head(); } @@ -176,7 +175,6 @@ impl Staker { } for vote in new_votes_to_add { - // TODO: proper error handling new_block .votes .push(vote) @@ -239,7 +237,7 @@ impl Staker { &state.latest_justified_slot ); - self.receive(&QueueItem::VoteItem(signed_vote.clone())); + self.receive(QueueItem::VoteItem(signed_vote)); // TODO: submit to actual network // self.get_network() @@ -248,7 +246,7 @@ impl Staker { } /// Called by the p2p network - fn receive(&mut self, queue_item: &QueueItem) { + fn receive(&mut self, queue_item: QueueItem) { match queue_item { QueueItem::BlockItem(block) => { let block_hash = block.tree_hash_root(); @@ -260,26 +258,24 @@ impl Staker { match self.post_states.get(&block.parent) { Some(parent_state) => { - let state = process_block(parent_state, block); - - self.chain.insert(block_hash, block.clone()); - self.post_states.insert(block_hash, state); - - let mut known_votes = self.known_votes.clone().into_iter(); + let state = process_block(parent_state, &block); for vote in &block.votes { - if !known_votes.any(|known_vote| known_vote == *vote) { + if !self.known_votes.contains(vote) { self.known_votes.push(vote.clone()); } } + self.chain.insert(block_hash, block); + self.post_states.insert(block_hash, state); + self.recompute_head(); // Once we have received a block, also process all of // its dependencies if let Some(queue_items) = self.dependencies.get(&block_hash) { for item in queue_items.clone() { - self.receive(&item); + self.receive(item); } self.dependencies.remove(&block_hash); @@ -291,32 +287,23 @@ impl Staker { self.dependencies .entry(block.parent) .or_default() - .push(queue_item.clone()); + .push(QueueItem::BlockItem(block)); } } } QueueItem::VoteItem(vote) => { - let is_known_vote = self - .known_votes - .clone() - .into_iter() - .any(|known_vote| known_vote == vote.data); - - let is_new_vote = self - .new_votes - .clone() - .into_iter() - .any(|new_vote| new_vote == vote.data); + let is_known_vote = self.known_votes.contains(&vote.data); + let is_new_vote = self.new_votes.contains(&vote.data); if is_known_vote || is_new_vote { // Do nothing } else if self.chain.contains_key(&vote.data.head) { - self.new_votes.push(vote.data.clone()); + self.new_votes.push(vote.data); } else { self.dependencies .entry(vote.data.head) .or_default() - .push(queue_item.clone()); + .push(QueueItem::VoteItem(vote)); } } } From 21c5742390229ab5659cb42283a1e3150f915ce3 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 14:35:30 +0700 Subject: [PATCH 38/47] refactor: one less clone() --- crates/common/lean_chain/src/staker.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/lean_chain/src/staker.rs index 8d6f7feb0..7426c7a72 100644 --- a/crates/common/lean_chain/src/staker.rs +++ b/crates/common/lean_chain/src/staker.rs @@ -271,14 +271,11 @@ impl Staker { self.recompute_head(); - // Once we have received a block, also process all of - // its dependencies - if let Some(queue_items) = self.dependencies.get(&block_hash) { - for item in queue_items.clone() { + // Once we have received a block, also process all of its dependencies + if let Some(queue_items) = self.dependencies.remove(&block_hash) { + for item in queue_items { self.receive(item); } - - self.dependencies.remove(&block_hash); } } None => { From 2243990349c2a1fd3b81c8243aa5e793af6ea2a3 Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 17:41:40 +0700 Subject: [PATCH 39/47] refactor: ream_lean_chain to ream_chain_lean --- Cargo.toml | 4 ++-- crates/common/{lean_chain => chain/lean}/Cargo.toml | 2 +- crates/common/{lean_chain => chain/lean}/src/lib.rs | 0 crates/common/{lean_chain => chain/lean}/src/staker.rs | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename crates/common/{lean_chain => chain/lean}/Cargo.toml (95%) rename crates/common/{lean_chain => chain/lean}/src/lib.rs (100%) rename crates/common/{lean_chain => chain/lean}/src/staker.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 422403807..cf346f4bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/account_manager", "crates/common/beacon_api_types", "crates/common/chain/beacon", + "crates/common/chain/lean", "crates/common/checkpoint_sync", "crates/common/consensus/beacon", "crates/common/consensus/lean", @@ -12,7 +13,6 @@ members = [ "crates/common/execution_engine", "crates/common/executor", "crates/common/fork_choice", - "crates/common/lean_chain", "crates/common/light_client", "crates/common/network_spec", "crates/common/node", @@ -118,6 +118,7 @@ ream-account-manager = { path = "crates/account_manager" } ream-beacon-api-types = { path = "crates/common/beacon_api_types" } ream-bls = { path = "crates/crypto/bls", features = ["zkcrypto"] } # Default feature is zkcrypto ream-chain-beacon = { path = "crates/common/chain/beacon" } +ream-chain-lean = { path = "crates/common/chain/lean" } ream-checkpoint-sync = { path = "crates/common/checkpoint_sync" } ream-consensus-beacon = { path = "crates/common/consensus/beacon" } ream-consensus-lean = { path = "crates/common/consensus/lean" } @@ -127,7 +128,6 @@ ream-execution-engine = { path = "crates/common/execution_engine" } ream-executor = { path = "crates/common/executor" } ream-fork-choice = { path = "crates/common/fork_choice" } ream-keystore = { path = "crates/crypto/keystore" } -ream-lean-chain = { path = "crates/common/lean_chain" } ream-light-client = { path = "crates/common/light_client" } ream-merkle = { path = "crates/crypto/merkle" } ream-network-manager = { path = "crates/networking/manager" } diff --git a/crates/common/lean_chain/Cargo.toml b/crates/common/chain/lean/Cargo.toml similarity index 95% rename from crates/common/lean_chain/Cargo.toml rename to crates/common/chain/lean/Cargo.toml index 124855407..c250a2ae8 100644 --- a/crates/common/lean_chain/Cargo.toml +++ b/crates/common/chain/lean/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "ream-lean-chain" +name = "ream-chain-lean" authors.workspace = true edition.workspace = true keywords.workspace = true diff --git a/crates/common/lean_chain/src/lib.rs b/crates/common/chain/lean/src/lib.rs similarity index 100% rename from crates/common/lean_chain/src/lib.rs rename to crates/common/chain/lean/src/lib.rs diff --git a/crates/common/lean_chain/src/staker.rs b/crates/common/chain/lean/src/staker.rs similarity index 100% rename from crates/common/lean_chain/src/staker.rs rename to crates/common/chain/lean/src/staker.rs From d62af71d1377a8e5edd8a7ab84ec0545ad0685ec Mon Sep 17 00:00:00 2001 From: unnawut Date: Mon, 4 Aug 2025 23:30:20 +0700 Subject: [PATCH 40/47] fix: proper justifications handling --- crates/common/consensus/lean/src/state.rs | 253 +++++++++++++++++++--- 1 file changed, 228 insertions(+), 25 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 3f04aa4be..37839d86f 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -3,15 +3,11 @@ use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{ BitList, VariableList, - typenum::{U262144, U1073741824, Unsigned}, + typenum::{U262144, U1073741824}, }; use tree_hash_derive::TreeHash; -use crate::{ - config::Config, - MAX_HISTORICAL_BLOCK_HASHES, - VALIDATOR_REGISTRY_LIMIT, -}; +use crate::{VALIDATOR_REGISTRY_LIMIT, config::Config}; #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash)] pub struct LeanState { @@ -34,15 +30,26 @@ pub struct LeanState { } impl LeanState { - fn get_justifications_roots_index(&self, root: &B256) -> Option { - self.justifications_roots.iter().position(|r| r == root) - } + pub fn new(num_validators: u64) -> LeanState { + LeanState { + config: Config { num_validators }, - fn get_justifications_roots_range(&self, index: &usize) -> (usize, usize) { - let start_range = index * MAX_HISTORICAL_BLOCK_HASHES as usize; - let end_range = start_range + VALIDATOR_REGISTRY_LIMIT as usize; + latest_justified_hash: B256::ZERO, + latest_justified_slot: 0, + latest_finalized_hash: B256::ZERO, + latest_finalized_slot: 0, - (start_range, end_range) + historical_block_hashes: VariableList::empty(), + justified_slots: VariableList::empty(), + + justifications_roots: VariableList::empty(), + justifications_roots_validators: BitList::with_capacity(0) + .expect("Failed to initialize state's justifications_roots_validators"), + } + } + + fn get_justifications_roots_index(&self, root: &B256) -> Option { + self.justifications_roots.iter().position(|r| r == root) } pub fn initialize_justifications_for_root(&mut self, root: &B256) { @@ -50,6 +57,26 @@ impl LeanState { self.justifications_roots .push(*root) .expect("Failed to insert root into justifications_roots"); + + let old_length = self.justifications_roots_validators.len(); + let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; + + let mut new_justifications_roots_validators = BitList::with_capacity(new_length) + .expect("Failed to initialize new justification bits"); + + for (i, bit) in self.justifications_roots_validators.iter().enumerate() { + new_justifications_roots_validators + .set(i, bit) + .expect("Failed to initialize justification bits to existing values"); + } + + for i in old_length..new_length { + new_justifications_roots_validators + .set(i, false) + .expect("Failed to zero-fill justification bits"); + } + + self.justifications_roots_validators = new_justifications_roots_validators; } } @@ -57,8 +84,12 @@ impl LeanState { let index = self .get_justifications_roots_index(root) .expect("Failed to find the justifications index to set"); + self.justifications_roots_validators - .set(index * U262144::to_usize() + *validator_id as usize, value) + .set( + index * VALIDATOR_REGISTRY_LIMIT as usize + *validator_id as usize, + value, + ) .expect("Failed to set justification bit"); } @@ -67,29 +98,201 @@ impl LeanState { .get_justifications_roots_index(root) .expect("Could not find justifications for the provided block root"); - let (start_range, end_range) = self.get_justifications_roots_range(&index); + let start_range = index * VALIDATOR_REGISTRY_LIMIT as usize; - self.justifications_roots_validators.as_slice()[start_range..end_range] + self.justifications_roots_validators .iter() + .skip(start_range) + .take(VALIDATOR_REGISTRY_LIMIT as usize) .fold(0, |acc, justification_bits| { - acc + justification_bits.count_ones() + acc + justification_bits as usize }) as u64 } pub fn remove_justifications(&mut self, root: &B256) { - // Remove from `state.justifications_roots` let index = self .get_justifications_roots_index(root) .expect("Failed to find the justifications index to remove"); self.justifications_roots.remove(index); - let (start_range, end_range) = self.get_justifications_roots_range(&index); + let new_length = self.justifications_roots.len() * VALIDATOR_REGISTRY_LIMIT as usize; + let mut new_justifications_roots_validators = + BitList::::with_capacity(new_length) + .expect("Failed to recreate state's justifications_roots_validators"); - // Remove from `state.justifications_roots_validators` - for i in start_range..end_range { - self.justifications_roots_validators - .set(i, false) - .expect("Failed to remove justifications"); - } + // Take left side of the list (if any) + self.justifications_roots_validators + .iter() + .take(index * VALIDATOR_REGISTRY_LIMIT as usize) + .fold(0, |i, justification_bit| { + new_justifications_roots_validators + .set(i, justification_bit) + .expect("Failed to set new justification bit"); + i + 1 + }); + + // Take right side of the list (if any) + self.justifications_roots_validators + .iter() + .skip((index + 1) * VALIDATOR_REGISTRY_LIMIT as usize) + .fold( + index * VALIDATOR_REGISTRY_LIMIT as usize, + |i, justification_bit| { + new_justifications_roots_validators + .set(i, justification_bit) + .expect("Failed to set new justification bit"); + i + 1 + }, + ); + + self.justifications_roots_validators = new_justifications_roots_validators; + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn initialize_justifications_for_root() { + let mut state = LeanState::new(1); + + // Initialize 1st root + state.initialize_justifications_for_root(&B256::repeat_byte(1)); + assert_eq!(state.justifications_roots.len(), 1); + assert_eq!( + state.justifications_roots_validators.len(), + VALIDATOR_REGISTRY_LIMIT as usize + ); + + // Initialize an existing root should result in same lengths + state.initialize_justifications_for_root(&B256::repeat_byte(1)); + assert_eq!(state.justifications_roots.len(), 1); + assert_eq!( + state.justifications_roots_validators.len(), + VALIDATOR_REGISTRY_LIMIT as usize + ); + + // Initialize 2nd root + state.initialize_justifications_for_root(&B256::repeat_byte(2)); + assert_eq!(state.justifications_roots.len(), 2); + assert_eq!( + state.justifications_roots_validators.len(), + 2 * VALIDATOR_REGISTRY_LIMIT as usize + ); + } + + #[test] + fn set_justification() { + let mut state = LeanState::new(1); + let root0 = B256::repeat_byte(1); + let root1 = B256::repeat_byte(2); + let validator_id = 7u64; + + // Set for 1st root + state.initialize_justifications_for_root(&root0); + state.set_justification(&root0, &validator_id, true); + assert!( + state + .justifications_roots_validators + .get(validator_id as usize) + .unwrap() + ); + + // Set for 2nd root + state.initialize_justifications_for_root(&root1); + state.set_justification(&root1, &validator_id, true); + assert!( + state + .justifications_roots_validators + .get(VALIDATOR_REGISTRY_LIMIT as usize + validator_id as usize) + .unwrap() + ); + } + + #[test] + fn count_justifications() { + let mut state = LeanState::new(1); + let root0 = B256::repeat_byte(1); + let root1 = B256::repeat_byte(2); + + // Justifications for 1st root, up to 2 justifications + state.initialize_justifications_for_root(&root0); + + state.set_justification(&root0, &1u64, true); + assert_eq!(state.count_justifications(&root0), 1); + + state.set_justification(&root0, &2u64, true); + assert_eq!(state.count_justifications(&root0), 2); + + // Justifications for 2nd root, up to 3 justifications + state.initialize_justifications_for_root(&root1); + + state.set_justification(&root1, &11u64, true); + assert_eq!(state.count_justifications(&root1), 1); + + state.set_justification(&root1, &22u64, true); + state.set_justification(&root1, &33u64, true); + assert_eq!(state.count_justifications(&root1), 3); + } + + #[test] + fn remove_justifications() { + // Assuming 3 roots & 4 validators + let mut state = LeanState::new(3); + let root0 = B256::repeat_byte(1); + let root1 = B256::repeat_byte(2); + let root2 = B256::repeat_byte(3); + + // Add justifications for left root + state.initialize_justifications_for_root(&root0); + state.set_justification(&root0, &0u64, true); + + // Add justifications for middle root + state.initialize_justifications_for_root(&root1); + state.set_justification(&root1, &1u64, true); + + // Add justifications for last root + state.initialize_justifications_for_root(&root2); + state.set_justification(&root2, &2u64, true); + + // Assert before removal + assert_eq!(state.justifications_roots.len(), 3); + assert_eq!( + state.justifications_roots_validators.len(), + 3 * VALIDATOR_REGISTRY_LIMIT as usize + ); + + // Assert after removing middle root (root1) + state.remove_justifications(&root1); + + assert_eq!( + state.get_justifications_roots_index(&root1), + None, + "Root still exists after removal" + ); + assert_eq!( + state.justifications_roots.len(), + 2, + "Should be reduced by 1" + ); + assert_eq!( + state.justifications_roots_validators.len(), + 2 * VALIDATOR_REGISTRY_LIMIT as usize, + "Should be reduced by VALIDATOR_REGISTRY_LIMIT" + ); + + // Assert justifications + assert!( + state.justifications_roots_validators.get(0).unwrap(), + "root0 should still be justified by validator0" + ); + assert!( + state + .justifications_roots_validators + .get(VALIDATOR_REGISTRY_LIMIT as usize + 2) + .unwrap(), + "root2 should still be justified by validator2" + ); } } From c85d6eaa1da9a31e48e30bde987d3253a00c3003 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 01:02:03 +0700 Subject: [PATCH 41/47] feat: use anyhow --- Cargo.lock | 34 +++--- crates/common/chain/lean/Cargo.toml | 1 + crates/common/chain/lean/src/staker.rs | 107 +++++++++++------- crates/common/consensus/lean/Cargo.toml | 1 + crates/common/consensus/lean/src/lib.rs | 48 ++++---- crates/common/consensus/lean/src/state.rs | 130 +++++++++++----------- 6 files changed, 183 insertions(+), 138 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c874f5d8b..7f9b7454c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5253,6 +5253,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "ream-chain-lean" +version = "0.1.0" +dependencies = [ + "alloy-primitives", + "anyhow", + "ream-consensus-lean", + "ream-network-spec", + "ream-p2p", + "ream-pqc", + "serde", + "ssz_types", + "tokio", + "tracing", + "tree_hash", +] + [[package]] name = "ream-checkpoint-sync" version = "0.1.0" @@ -5309,6 +5326,7 @@ name = "ream-consensus-lean" version = "0.1.0" dependencies = [ "alloy-primitives", + "anyhow", "ethereum_hashing", "ethereum_ssz", "ethereum_ssz_derive", @@ -5443,22 +5461,6 @@ dependencies = [ "ssz_types", ] -[[package]] -name = "ream-lean-chain" -version = "0.1.0" -dependencies = [ - "alloy-primitives", - "ream-consensus-lean", - "ream-network-spec", - "ream-p2p", - "ream-pqc", - "serde", - "ssz_types", - "tokio", - "tracing", - "tree_hash", -] - [[package]] name = "ream-light-client" version = "0.1.0" diff --git a/crates/common/chain/lean/Cargo.toml b/crates/common/chain/lean/Cargo.toml index c250a2ae8..c822444c5 100644 --- a/crates/common/chain/lean/Cargo.toml +++ b/crates/common/chain/lean/Cargo.toml @@ -10,6 +10,7 @@ rust-version.workspace = true version.workspace = true [dependencies] +anyhow.workspace = true alloy-primitives.workspace = true serde.workspace = true ssz_types.workspace = true diff --git a/crates/common/chain/lean/src/staker.rs b/crates/common/chain/lean/src/staker.rs index 7426c7a72..825f24d45 100644 --- a/crates/common/chain/lean/src/staker.rs +++ b/crates/common/chain/lean/src/staker.rs @@ -77,8 +77,9 @@ impl Staker { } /// Compute the latest block that the staker is allowed to choose as the target - fn compute_safe_target(&self) -> B256 { - let justified_hash = get_latest_justified_hash(&self.post_states).unwrap(); + fn compute_safe_target(&self) -> anyhow::Result { + let justified_hash = get_latest_justified_hash(&self.post_states) + .ok_or_else(|| anyhow::anyhow!("No justified hash found in post states"))?; get_fork_choice_head( &self.chain, @@ -90,65 +91,80 @@ impl Staker { /// Process new votes that the staker has received. Vote processing is done /// at a particular time, because of safe target and view merge rule - fn accept_new_votes(&mut self) { + fn accept_new_votes(&mut self) -> anyhow::Result<()> { for new_vote in self.new_votes.drain(..) { if !self.known_votes.contains(&new_vote) { self.known_votes.push(new_vote); } } - self.recompute_head(); + self.recompute_head()?; + Ok(()) } /// Done upon processing new votes or a new block - fn recompute_head(&mut self) { + fn recompute_head(&mut self) -> anyhow::Result<()> { let justified_hash = get_latest_justified_hash(&self.post_states) - .expect("Failed to get latest_justified_hash from post_states"); - self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0); + .ok_or_else(|| anyhow::anyhow!("Failed to get latest_justified_hash from post_states"))?; + self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0)?; + Ok(()) } /// Called every second - pub fn tick(&mut self) { - let time_in_slot = self.network.lock().unwrap().time % SLOT_DURATION; + pub fn tick(&mut self) -> anyhow::Result<()> { + let current_slot = self.get_current_slot()?; + let time_in_slot = { + let network = self.network.lock() + .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; + network.time % SLOT_DURATION + }; // t=0: propose a block if time_in_slot == 0 { - if self.get_current_slot() % self.num_validators == self.validator_id { + if current_slot % self.num_validators == self.validator_id { // View merge mechanism: a node accepts attestations that it received // <= 1/4 before slot start, or attestations in the latest block - self.accept_new_votes(); - self.propose_block(); + self.accept_new_votes()?; + self.propose_block()?; } // t=1/4: vote } else if time_in_slot == SLOT_DURATION / 4 { - self.vote(); + self.vote()?; // t=2/4: compute the safe target (this must be done here to ensure // that, assuming network latency assumptions are satisfied, anything that // one honest node receives by this time, every honest node will receive by // the general attestation deadline) } else if time_in_slot == SLOT_DURATION * 2 / 4 { - self.safe_target = self.compute_safe_target(); + self.safe_target = self.compute_safe_target()?; // Deadline to accept attestations except for those included in a block } else if time_in_slot == SLOT_DURATION * 3 / 4 { - self.accept_new_votes(); + self.accept_new_votes()?; } + + Ok(()) } - fn get_current_slot(&self) -> u64 { - self.network.lock().unwrap().time / SLOT_DURATION + 2 + fn get_current_slot(&self) -> anyhow::Result { + let network = self.network.lock() + .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; + Ok(network.time / SLOT_DURATION + 2) } /// Called when it's the staker's turn to propose a block - fn propose_block(&mut self) { - let new_slot = self.get_current_slot(); + fn propose_block(&mut self) -> anyhow::Result<()> { + let new_slot = self.get_current_slot()?; + + let head_block = self.chain.get(&self.head) + .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; info!( "proposing (Staker {}), head = {}", self.validator_id, - self.chain.get(&self.head).unwrap().slot + head_block.slot ); - let head_state = self.post_states.get(&self.head).unwrap(); + let head_state = self.post_states.get(&self.head) + .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; let mut new_block = Block { slot: new_slot, parent: self.head, @@ -160,7 +176,7 @@ impl Staker { // Keep attempt to add valid votes from the list of available votes loop { - state = process_block(head_state, &new_block); + state = process_block(head_state, &new_block)?; let new_votes_to_add = self .known_votes @@ -178,7 +194,7 @@ impl Staker { new_block .votes .push(vote) - .expect("Failed to add vote to new_block"); + .map_err(|e| anyhow::anyhow!("Failed to add vote to new_block: {:?}", e))?; } } @@ -192,32 +208,43 @@ impl Staker { // self.get_network() // .borrow_mut() // .submit(QueueItem::BlockItem(new_block), self.validator_id); + + Ok(()) } /// Called when it's the staker's turn to vote - fn vote(&mut self) { - let state = self.post_states.get(&self.head).unwrap(); - let mut target_block = self.chain.get(&self.head).unwrap(); + fn vote(&mut self) -> anyhow::Result<()> { + let state = self.post_states.get(&self.head) + .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; + let mut target_block = self.chain.get(&self.head) + .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; // If there is no very recent safe target, then vote for the k'th ancestor // of the head for _ in 0..3 { - if target_block.slot > self.chain.get(&self.safe_target).unwrap().slot { - target_block = self.chain.get(&target_block.parent).unwrap(); + let safe_target_block = self.chain.get(&self.safe_target) + .ok_or_else(|| anyhow::anyhow!("Safe target block not found for hash: {}", self.safe_target))?; + if target_block.slot > safe_target_block.slot { + target_block = self.chain.get(&target_block.parent) + .ok_or_else(|| anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent))?; } } // If the latest finalized slot is very far back, then only some slots are // valid to justify, make sure the target is one of those while !is_justifiable_slot(&state.latest_finalized_slot, &target_block.slot) { - target_block = self.chain.get(&target_block.parent).unwrap(); + target_block = self.chain.get(&target_block.parent) + .ok_or_else(|| anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent))?; } + let head_block = self.chain.get(&self.head) + .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; + let vote = Vote { validator_id: self.validator_id, - slot: self.get_current_slot(), + slot: self.get_current_slot()?, head: self.head, - head_slot: self.chain.get(&self.head).unwrap().slot, + head_slot: head_block.slot, target: target_block.tree_hash_root(), target_slot: target_block.slot, source: state.latest_justified_hash, @@ -232,33 +259,35 @@ impl Staker { info!( "voting (Staker {}), head = {}, t = {}, s = {}", self.validator_id, - &self.chain.get(&self.head).unwrap().slot, + &head_block.slot, &target_block.slot, &state.latest_justified_slot ); - self.receive(QueueItem::VoteItem(signed_vote)); + self.receive(QueueItem::VoteItem(signed_vote))?; // TODO: submit to actual network // self.get_network() // .borrow_mut() // .submit(QueueItem::VoteItem(vote), self.validator_id); + + Ok(()) } /// Called by the p2p network - fn receive(&mut self, queue_item: QueueItem) { + fn receive(&mut self, queue_item: QueueItem) -> anyhow::Result<()> { match queue_item { QueueItem::BlockItem(block) => { let block_hash = block.tree_hash_root(); // If the block is already known, ignore it if self.chain.contains_key(&block_hash) { - return; + return Ok(()); } match self.post_states.get(&block.parent) { Some(parent_state) => { - let state = process_block(parent_state, &block); + let state = process_block(parent_state, &block)?; for vote in &block.votes { if !self.known_votes.contains(vote) { @@ -269,12 +298,12 @@ impl Staker { self.chain.insert(block_hash, block); self.post_states.insert(block_hash, state); - self.recompute_head(); + self.recompute_head()?; // Once we have received a block, also process all of its dependencies if let Some(queue_items) = self.dependencies.remove(&block_hash) { for item in queue_items { - self.receive(item); + self.receive(item)?; } } } @@ -304,5 +333,7 @@ impl Staker { } } } + + Ok(()) } } diff --git a/crates/common/consensus/lean/Cargo.toml b/crates/common/consensus/lean/Cargo.toml index 4ab7dc944..ba10bd7be 100644 --- a/crates/common/consensus/lean/Cargo.toml +++ b/crates/common/consensus/lean/Cargo.toml @@ -11,6 +11,7 @@ version.workspace = true [dependencies] alloy-primitives.workspace = true +anyhow.workspace = true ethereum_hashing.workspace = true ethereum_ssz.workspace = true ethereum_ssz_derive.workspace = true diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 631943d1f..52774c330 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -3,10 +3,10 @@ pub mod config; pub mod state; pub mod vote; -use std::collections::HashMap; - use alloy_primitives::B256; +use anyhow::anyhow; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use crate::{ block::Block, @@ -41,30 +41,30 @@ pub fn is_justifiable_slot(finalized_slot: &u64, candidate_slot: &u64) -> bool { } /// Given a state, output the new state after processing that block -pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { +pub fn process_block(pre_state: &LeanState, block: &Block) -> anyhow::Result { let mut state = pre_state.clone(); // Track historical blocks in the state state .historical_block_hashes .push(block.parent) - .expect("Failed to add block.parent to historical_block_hashes"); + .map_err(|err| anyhow!("Failed to add block.parent to historical_block_hashes: {err:?}"))?; state .justified_slots .push(false) - .expect("Failed to add to justified_slots"); + .map_err(|err| anyhow!("Failed to add to justified_slots: {err:?}"))?; while state.historical_block_hashes.len() < block.slot as usize { state .justified_slots .push(false) - .expect("Failed to prefill justified_slots"); + .map_err(|err| anyhow!("Failed to prefill justified_slots: {err:?}"))?; state .historical_block_hashes // Diverged from Python implementation: uses `B256::ZERO` instead of `None` .push(B256::ZERO) - .expect("Failed to prefill historical_block_hashes"); + .map_err(|err| anyhow!("Failed to prefill historical_block_hashes: {err:?}"))?; } // Process votes @@ -82,10 +82,10 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { } // Track attempts to justify new hashes - state.initialize_justifications_for_root(&vote.target); - state.set_justification(&vote.target, &vote.validator_id, true); + state.initialize_justifications_for_root(&vote.target)?; + state.set_justification(&vote.target, &vote.validator_id, true)?; - let count = state.count_justifications(&vote.target); + let count = state.count_justifications(&vote.target)?; // If 2/3 voted for the same new valid hash to justify if count == (2 * state.config.num_validators) / 3 { @@ -93,7 +93,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { state.latest_justified_slot = vote.target_slot; state.justified_slots[vote.target_slot as usize] = true; - state.remove_justifications(&vote.target); + state.remove_justifications(&vote.target)?; // Finalization: if the target is the next valid justifiable // hash after the source @@ -107,7 +107,7 @@ pub fn process_block(pre_state: &LeanState, block: &Block) -> LeanState { } } - state + Ok(state) } /// Get the highest-slot justified block that we know about @@ -125,7 +125,7 @@ pub fn get_fork_choice_head( provided_root: &B256, votes: &[Vote], min_score: u64, -) -> B256 { +) -> anyhow::Result { let mut root = *provided_root; // Start at genesis by default @@ -134,7 +134,7 @@ pub fn get_fork_choice_head( .iter() .min_by_key(|(_, block)| block.slot) .map(|(hash, _)| *hash) - .unwrap(); + .ok_or_else(|| anyhow!("No blocks found to determine genesis"))?; } // Identify latest votes @@ -158,10 +158,18 @@ pub fn get_fork_choice_head( for vote in latest_votes.values() { if blocks.contains_key(&vote.head) { let mut block_hash = vote.head; - while blocks.get(&block_hash).unwrap().slot > blocks.get(&root).unwrap().slot { + while { + let current_block = blocks.get(&block_hash) + .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; + let root_block = blocks.get(&root) + .ok_or_else(|| anyhow!("Root block not found for hash: {}", root))?; + current_block.slot > root_block.slot + } { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); vote_weights.insert(block_hash, current_weights + 1); - block_hash = blocks.get(&block_hash).unwrap().parent; + let current_block = blocks.get(&block_hash) + .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; + block_hash = current_block.parent; } } } @@ -184,17 +192,19 @@ pub fn get_fork_choice_head( loop { match children_map.get(¤t_root) { None => { - break current_root; + break Ok(current_root); } Some(children) => { current_root = *children .iter() .max_by_key(|child_hash| { let vote_weight = vote_weights.get(*child_hash).unwrap_or(&0); - let slot = blocks.get(*child_hash).unwrap().slot; + let slot = blocks.get(*child_hash) + .map(|block| block.slot) + .unwrap_or(0); (*vote_weight, slot, *(*child_hash)) }) - .unwrap(); + .ok_or_else(|| anyhow!("No children found for current root: {}", current_root))?; } } } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 37839d86f..8d9c1f437 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -1,4 +1,5 @@ use alloy_primitives::B256; +use anyhow::anyhow; use serde::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::{ @@ -30,8 +31,8 @@ pub struct LeanState { } impl LeanState { - pub fn new(num_validators: u64) -> LeanState { - LeanState { + pub fn new(num_validators: u64) -> anyhow::Result { + Ok(LeanState { config: Config { num_validators }, latest_justified_hash: B256::ZERO, @@ -44,108 +45,107 @@ impl LeanState { justifications_roots: VariableList::empty(), justifications_roots_validators: BitList::with_capacity(0) - .expect("Failed to initialize state's justifications_roots_validators"), - } + .map_err(|err| anyhow!("Failed to initialize state's justifications_roots_validators: {err:?}"))?, + }) } fn get_justifications_roots_index(&self, root: &B256) -> Option { self.justifications_roots.iter().position(|r| r == root) } - pub fn initialize_justifications_for_root(&mut self, root: &B256) { + pub fn initialize_justifications_for_root(&mut self, root: &B256) -> anyhow::Result<()> { if !self.justifications_roots.contains(root) { self.justifications_roots .push(*root) - .expect("Failed to insert root into justifications_roots"); + .map_err(|err| anyhow!("Failed to insert root into justifications_roots: {err:?}"))?; let old_length = self.justifications_roots_validators.len(); let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; let mut new_justifications_roots_validators = BitList::with_capacity(new_length) - .expect("Failed to initialize new justification bits"); + .map_err(|err| anyhow!("Failed to initialize new justification bits: {err:?}"))?; for (i, bit) in self.justifications_roots_validators.iter().enumerate() { new_justifications_roots_validators .set(i, bit) - .expect("Failed to initialize justification bits to existing values"); + .map_err(|err| anyhow!("Failed to initialize justification bits to existing values: {err:?}"))?; } for i in old_length..new_length { new_justifications_roots_validators .set(i, false) - .expect("Failed to zero-fill justification bits"); + .map_err(|err| anyhow!("Failed to zero-fill justification bits: {err:?}"))?; } self.justifications_roots_validators = new_justifications_roots_validators; } + Ok(()) } - pub fn set_justification(&mut self, root: &B256, validator_id: &u64, value: bool) { + pub fn set_justification(&mut self, root: &B256, validator_id: &u64, value: bool) -> anyhow::Result<()> { let index = self .get_justifications_roots_index(root) - .expect("Failed to find the justifications index to set"); + .ok_or(anyhow!("Failed to find the justifications index to set"))?; self.justifications_roots_validators .set( index * VALIDATOR_REGISTRY_LIMIT as usize + *validator_id as usize, value, ) - .expect("Failed to set justification bit"); + .map_err(|err| anyhow!("Failed to set justification bit: {err:?}"))?; + + Ok(()) } - pub fn count_justifications(&self, root: &B256) -> u64 { + pub fn count_justifications(&self, root: &B256) -> anyhow::Result { let index = self .get_justifications_roots_index(root) - .expect("Could not find justifications for the provided block root"); + .ok_or_else(|| anyhow!("Could not find justifications for the provided block root"))?; let start_range = index * VALIDATOR_REGISTRY_LIMIT as usize; - self.justifications_roots_validators + Ok(self.justifications_roots_validators .iter() .skip(start_range) .take(VALIDATOR_REGISTRY_LIMIT as usize) .fold(0, |acc, justification_bits| { acc + justification_bits as usize - }) as u64 + }) as u64) } - pub fn remove_justifications(&mut self, root: &B256) { + pub fn remove_justifications(&mut self, root: &B256) -> anyhow::Result<()> { let index = self .get_justifications_roots_index(root) - .expect("Failed to find the justifications index to remove"); + .ok_or_else(|| anyhow!("Failed to find the justifications index to remove"))?; self.justifications_roots.remove(index); let new_length = self.justifications_roots.len() * VALIDATOR_REGISTRY_LIMIT as usize; let mut new_justifications_roots_validators = BitList::::with_capacity(new_length) - .expect("Failed to recreate state's justifications_roots_validators"); + .map_err(|e| anyhow!("Failed to recreate state's justifications_roots_validators: {:?}", e))?; // Take left side of the list (if any) - self.justifications_roots_validators + for (i, justification_bit) in self.justifications_roots_validators .iter() .take(index * VALIDATOR_REGISTRY_LIMIT as usize) - .fold(0, |i, justification_bit| { - new_justifications_roots_validators - .set(i, justification_bit) - .expect("Failed to set new justification bit"); - i + 1 - }); + .enumerate() { + new_justifications_roots_validators + .set(i, justification_bit) + .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; + } // Take right side of the list (if any) - self.justifications_roots_validators + for (i, justification_bit) in self.justifications_roots_validators .iter() .skip((index + 1) * VALIDATOR_REGISTRY_LIMIT as usize) - .fold( - index * VALIDATOR_REGISTRY_LIMIT as usize, - |i, justification_bit| { - new_justifications_roots_validators - .set(i, justification_bit) - .expect("Failed to set new justification bit"); - i + 1 - }, - ); + .enumerate() { + new_justifications_roots_validators + .set(index * VALIDATOR_REGISTRY_LIMIT as usize + i, justification_bit) + .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; + } self.justifications_roots_validators = new_justifications_roots_validators; + Ok(()) } } @@ -155,10 +155,10 @@ mod test { #[test] fn initialize_justifications_for_root() { - let mut state = LeanState::new(1); + let mut state = LeanState::new(1).unwrap(); // Initialize 1st root - state.initialize_justifications_for_root(&B256::repeat_byte(1)); + state.initialize_justifications_for_root(&B256::repeat_byte(1)).unwrap(); assert_eq!(state.justifications_roots.len(), 1); assert_eq!( state.justifications_roots_validators.len(), @@ -166,7 +166,7 @@ mod test { ); // Initialize an existing root should result in same lengths - state.initialize_justifications_for_root(&B256::repeat_byte(1)); + state.initialize_justifications_for_root(&B256::repeat_byte(1)).unwrap(); assert_eq!(state.justifications_roots.len(), 1); assert_eq!( state.justifications_roots_validators.len(), @@ -174,7 +174,7 @@ mod test { ); // Initialize 2nd root - state.initialize_justifications_for_root(&B256::repeat_byte(2)); + state.initialize_justifications_for_root(&B256::repeat_byte(2)).unwrap(); assert_eq!(state.justifications_roots.len(), 2); assert_eq!( state.justifications_roots_validators.len(), @@ -184,14 +184,14 @@ mod test { #[test] fn set_justification() { - let mut state = LeanState::new(1); + let mut state = LeanState::new(1).unwrap(); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); let validator_id = 7u64; // Set for 1st root - state.initialize_justifications_for_root(&root0); - state.set_justification(&root0, &validator_id, true); + state.initialize_justifications_for_root(&root0).unwrap(); + state.set_justification(&root0, &validator_id, true).unwrap(); assert!( state .justifications_roots_validators @@ -200,8 +200,8 @@ mod test { ); // Set for 2nd root - state.initialize_justifications_for_root(&root1); - state.set_justification(&root1, &validator_id, true); + state.initialize_justifications_for_root(&root1).unwrap(); + state.set_justification(&root1, &validator_id, true).unwrap(); assert!( state .justifications_roots_validators @@ -212,49 +212,49 @@ mod test { #[test] fn count_justifications() { - let mut state = LeanState::new(1); + let mut state = LeanState::new(1).unwrap(); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); // Justifications for 1st root, up to 2 justifications - state.initialize_justifications_for_root(&root0); + state.initialize_justifications_for_root(&root0).unwrap(); - state.set_justification(&root0, &1u64, true); - assert_eq!(state.count_justifications(&root0), 1); + state.set_justification(&root0, &1u64, true).unwrap(); + assert_eq!(state.count_justifications(&root0).unwrap(), 1); - state.set_justification(&root0, &2u64, true); - assert_eq!(state.count_justifications(&root0), 2); + state.set_justification(&root0, &2u64, true).unwrap(); + assert_eq!(state.count_justifications(&root0).unwrap(), 2); // Justifications for 2nd root, up to 3 justifications - state.initialize_justifications_for_root(&root1); + state.initialize_justifications_for_root(&root1).unwrap(); - state.set_justification(&root1, &11u64, true); - assert_eq!(state.count_justifications(&root1), 1); + state.set_justification(&root1, &11u64, true).unwrap(); + assert_eq!(state.count_justifications(&root1).unwrap(), 1); - state.set_justification(&root1, &22u64, true); - state.set_justification(&root1, &33u64, true); - assert_eq!(state.count_justifications(&root1), 3); + state.set_justification(&root1, &22u64, true).unwrap(); + state.set_justification(&root1, &33u64, true).unwrap(); + assert_eq!(state.count_justifications(&root1).unwrap(), 3); } #[test] fn remove_justifications() { // Assuming 3 roots & 4 validators - let mut state = LeanState::new(3); + let mut state = LeanState::new(3).unwrap(); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); let root2 = B256::repeat_byte(3); // Add justifications for left root - state.initialize_justifications_for_root(&root0); - state.set_justification(&root0, &0u64, true); + state.initialize_justifications_for_root(&root0).unwrap(); + state.set_justification(&root0, &0u64, true).unwrap(); // Add justifications for middle root - state.initialize_justifications_for_root(&root1); - state.set_justification(&root1, &1u64, true); + state.initialize_justifications_for_root(&root1).unwrap(); + state.set_justification(&root1, &1u64, true).unwrap(); // Add justifications for last root - state.initialize_justifications_for_root(&root2); - state.set_justification(&root2, &2u64, true); + state.initialize_justifications_for_root(&root2).unwrap(); + state.set_justification(&root2, &2u64, true).unwrap(); // Assert before removal assert_eq!(state.justifications_roots.len(), 3); @@ -264,7 +264,7 @@ mod test { ); // Assert after removing middle root (root1) - state.remove_justifications(&root1); + state.remove_justifications(&root1).unwrap(); assert_eq!( state.get_justifications_roots_index(&root1), From 15f3b0069ce567b6dd370d605deda8983ce1f566 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 01:03:21 +0700 Subject: [PATCH 42/47] chore: make pr --- crates/common/chain/lean/src/staker.rs | 56 +++++++++++------- crates/common/consensus/lean/src/lib.rs | 20 ++++--- crates/common/consensus/lean/src/state.rs | 72 ++++++++++++++++------- 3 files changed, 99 insertions(+), 49 deletions(-) diff --git a/crates/common/chain/lean/src/staker.rs b/crates/common/chain/lean/src/staker.rs index 825f24d45..e0a84c078 100644 --- a/crates/common/chain/lean/src/staker.rs +++ b/crates/common/chain/lean/src/staker.rs @@ -104,8 +104,9 @@ impl Staker { /// Done upon processing new votes or a new block fn recompute_head(&mut self) -> anyhow::Result<()> { - let justified_hash = get_latest_justified_hash(&self.post_states) - .ok_or_else(|| anyhow::anyhow!("Failed to get latest_justified_hash from post_states"))?; + let justified_hash = get_latest_justified_hash(&self.post_states).ok_or_else(|| { + anyhow::anyhow!("Failed to get latest_justified_hash from post_states") + })?; self.head = get_fork_choice_head(&self.chain, &justified_hash, &self.known_votes, 0)?; Ok(()) } @@ -114,7 +115,9 @@ impl Staker { pub fn tick(&mut self) -> anyhow::Result<()> { let current_slot = self.get_current_slot()?; let time_in_slot = { - let network = self.network.lock() + let network = self + .network + .lock() .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; network.time % SLOT_DURATION }; @@ -145,7 +148,9 @@ impl Staker { } fn get_current_slot(&self) -> anyhow::Result { - let network = self.network.lock() + let network = self + .network + .lock() .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; Ok(network.time / SLOT_DURATION + 2) } @@ -154,16 +159,19 @@ impl Staker { fn propose_block(&mut self) -> anyhow::Result<()> { let new_slot = self.get_current_slot()?; - let head_block = self.chain.get(&self.head) + let head_block = self + .chain + .get(&self.head) .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; info!( "proposing (Staker {}), head = {}", - self.validator_id, - head_block.slot + self.validator_id, head_block.slot ); - let head_state = self.post_states.get(&self.head) + let head_state = self + .post_states + .get(&self.head) .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; let mut new_block = Block { slot: new_slot, @@ -214,30 +222,39 @@ impl Staker { /// Called when it's the staker's turn to vote fn vote(&mut self) -> anyhow::Result<()> { - let state = self.post_states.get(&self.head) + let state = self + .post_states + .get(&self.head) .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; - let mut target_block = self.chain.get(&self.head) + let mut target_block = self + .chain + .get(&self.head) .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; // If there is no very recent safe target, then vote for the k'th ancestor // of the head for _ in 0..3 { - let safe_target_block = self.chain.get(&self.safe_target) - .ok_or_else(|| anyhow::anyhow!("Safe target block not found for hash: {}", self.safe_target))?; + let safe_target_block = self.chain.get(&self.safe_target).ok_or_else(|| { + anyhow::anyhow!("Safe target block not found for hash: {}", self.safe_target) + })?; if target_block.slot > safe_target_block.slot { - target_block = self.chain.get(&target_block.parent) - .ok_or_else(|| anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent))?; + target_block = self.chain.get(&target_block.parent).ok_or_else(|| { + anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent) + })?; } } // If the latest finalized slot is very far back, then only some slots are // valid to justify, make sure the target is one of those while !is_justifiable_slot(&state.latest_finalized_slot, &target_block.slot) { - target_block = self.chain.get(&target_block.parent) - .ok_or_else(|| anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent))?; + target_block = self.chain.get(&target_block.parent).ok_or_else(|| { + anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent) + })?; } - let head_block = self.chain.get(&self.head) + let head_block = self + .chain + .get(&self.head) .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; let vote = Vote { @@ -258,10 +275,7 @@ impl Staker { info!( "voting (Staker {}), head = {}, t = {}, s = {}", - self.validator_id, - &head_block.slot, - &target_block.slot, - &state.latest_justified_slot + self.validator_id, &head_block.slot, &target_block.slot, &state.latest_justified_slot ); self.receive(QueueItem::VoteItem(signed_vote))?; diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 52774c330..007322f75 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -3,10 +3,11 @@ pub mod config; pub mod state; pub mod vote; +use std::collections::HashMap; + use alloy_primitives::B256; use anyhow::anyhow; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use crate::{ block::Block, @@ -159,15 +160,18 @@ pub fn get_fork_choice_head( if blocks.contains_key(&vote.head) { let mut block_hash = vote.head; while { - let current_block = blocks.get(&block_hash) + let current_block = blocks + .get(&block_hash) .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; - let root_block = blocks.get(&root) + let root_block = blocks + .get(&root) .ok_or_else(|| anyhow!("Root block not found for hash: {}", root))?; current_block.slot > root_block.slot } { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); vote_weights.insert(block_hash, current_weights + 1); - let current_block = blocks.get(&block_hash) + let current_block = blocks + .get(&block_hash) .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; block_hash = current_block.parent; } @@ -199,12 +203,12 @@ pub fn get_fork_choice_head( .iter() .max_by_key(|child_hash| { let vote_weight = vote_weights.get(*child_hash).unwrap_or(&0); - let slot = blocks.get(*child_hash) - .map(|block| block.slot) - .unwrap_or(0); + let slot = blocks.get(*child_hash).map(|block| block.slot).unwrap_or(0); (*vote_weight, slot, *(*child_hash)) }) - .ok_or_else(|| anyhow!("No children found for current root: {}", current_root))?; + .ok_or_else(|| { + anyhow!("No children found for current root: {}", current_root) + })?; } } } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 8d9c1f437..a4b0a3552 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -44,8 +44,9 @@ impl LeanState { justified_slots: VariableList::empty(), justifications_roots: VariableList::empty(), - justifications_roots_validators: BitList::with_capacity(0) - .map_err(|err| anyhow!("Failed to initialize state's justifications_roots_validators: {err:?}"))?, + justifications_roots_validators: BitList::with_capacity(0).map_err(|err| { + anyhow!("Failed to initialize state's justifications_roots_validators: {err:?}") + })?, }) } @@ -55,9 +56,9 @@ impl LeanState { pub fn initialize_justifications_for_root(&mut self, root: &B256) -> anyhow::Result<()> { if !self.justifications_roots.contains(root) { - self.justifications_roots - .push(*root) - .map_err(|err| anyhow!("Failed to insert root into justifications_roots: {err:?}"))?; + self.justifications_roots.push(*root).map_err(|err| { + anyhow!("Failed to insert root into justifications_roots: {err:?}") + })?; let old_length = self.justifications_roots_validators.len(); let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; @@ -68,7 +69,11 @@ impl LeanState { for (i, bit) in self.justifications_roots_validators.iter().enumerate() { new_justifications_roots_validators .set(i, bit) - .map_err(|err| anyhow!("Failed to initialize justification bits to existing values: {err:?}"))?; + .map_err(|err| { + anyhow!( + "Failed to initialize justification bits to existing values: {err:?}" + ) + })?; } for i in old_length..new_length { @@ -82,7 +87,12 @@ impl LeanState { Ok(()) } - pub fn set_justification(&mut self, root: &B256, validator_id: &u64, value: bool) -> anyhow::Result<()> { + pub fn set_justification( + &mut self, + root: &B256, + validator_id: &u64, + value: bool, + ) -> anyhow::Result<()> { let index = self .get_justifications_roots_index(root) .ok_or(anyhow!("Failed to find the justifications index to set"))?; @@ -104,7 +114,8 @@ impl LeanState { let start_range = index * VALIDATOR_REGISTRY_LIMIT as usize; - Ok(self.justifications_roots_validators + Ok(self + .justifications_roots_validators .iter() .skip(start_range) .take(VALIDATOR_REGISTRY_LIMIT as usize) @@ -121,26 +132,37 @@ impl LeanState { let new_length = self.justifications_roots.len() * VALIDATOR_REGISTRY_LIMIT as usize; let mut new_justifications_roots_validators = - BitList::::with_capacity(new_length) - .map_err(|e| anyhow!("Failed to recreate state's justifications_roots_validators: {:?}", e))?; + BitList::::with_capacity(new_length).map_err(|e| { + anyhow!( + "Failed to recreate state's justifications_roots_validators: {:?}", + e + ) + })?; // Take left side of the list (if any) - for (i, justification_bit) in self.justifications_roots_validators + for (i, justification_bit) in self + .justifications_roots_validators .iter() .take(index * VALIDATOR_REGISTRY_LIMIT as usize) - .enumerate() { + .enumerate() + { new_justifications_roots_validators .set(i, justification_bit) .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; } // Take right side of the list (if any) - for (i, justification_bit) in self.justifications_roots_validators + for (i, justification_bit) in self + .justifications_roots_validators .iter() .skip((index + 1) * VALIDATOR_REGISTRY_LIMIT as usize) - .enumerate() { + .enumerate() + { new_justifications_roots_validators - .set(index * VALIDATOR_REGISTRY_LIMIT as usize + i, justification_bit) + .set( + index * VALIDATOR_REGISTRY_LIMIT as usize + i, + justification_bit, + ) .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; } @@ -158,7 +180,9 @@ mod test { let mut state = LeanState::new(1).unwrap(); // Initialize 1st root - state.initialize_justifications_for_root(&B256::repeat_byte(1)).unwrap(); + state + .initialize_justifications_for_root(&B256::repeat_byte(1)) + .unwrap(); assert_eq!(state.justifications_roots.len(), 1); assert_eq!( state.justifications_roots_validators.len(), @@ -166,7 +190,9 @@ mod test { ); // Initialize an existing root should result in same lengths - state.initialize_justifications_for_root(&B256::repeat_byte(1)).unwrap(); + state + .initialize_justifications_for_root(&B256::repeat_byte(1)) + .unwrap(); assert_eq!(state.justifications_roots.len(), 1); assert_eq!( state.justifications_roots_validators.len(), @@ -174,7 +200,9 @@ mod test { ); // Initialize 2nd root - state.initialize_justifications_for_root(&B256::repeat_byte(2)).unwrap(); + state + .initialize_justifications_for_root(&B256::repeat_byte(2)) + .unwrap(); assert_eq!(state.justifications_roots.len(), 2); assert_eq!( state.justifications_roots_validators.len(), @@ -191,7 +219,9 @@ mod test { // Set for 1st root state.initialize_justifications_for_root(&root0).unwrap(); - state.set_justification(&root0, &validator_id, true).unwrap(); + state + .set_justification(&root0, &validator_id, true) + .unwrap(); assert!( state .justifications_roots_validators @@ -201,7 +231,9 @@ mod test { // Set for 2nd root state.initialize_justifications_for_root(&root1).unwrap(); - state.set_justification(&root1, &validator_id, true).unwrap(); + state + .set_justification(&root1, &validator_id, true) + .unwrap(); assert!( state .justifications_roots_validators From ba481ae2f1149e3bff4715be4b0c0d30c93bf2a1 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 01:09:23 +0700 Subject: [PATCH 43/47] fix: deps sort --- crates/common/chain/lean/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/common/chain/lean/Cargo.toml b/crates/common/chain/lean/Cargo.toml index c822444c5..625304667 100644 --- a/crates/common/chain/lean/Cargo.toml +++ b/crates/common/chain/lean/Cargo.toml @@ -10,8 +10,8 @@ rust-version.workspace = true version.workspace = true [dependencies] -anyhow.workspace = true alloy-primitives.workspace = true +anyhow.workspace = true serde.workspace = true ssz_types.workspace = true tokio.workspace = true From 214bb6005aea4cb3995ccb4622b50b2a41e712b5 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 01:30:16 +0700 Subject: [PATCH 44/47] chore: better error messages --- crates/common/consensus/lean/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index 007322f75..df7cdd701 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -135,7 +135,7 @@ pub fn get_fork_choice_head( .iter() .min_by_key(|(_, block)| block.slot) .map(|(hash, _)| *hash) - .ok_or_else(|| anyhow!("No blocks found to determine genesis"))?; + .ok_or_else(|| anyhow!("No blocks found to calculate fork choice"))?; } // Identify latest votes @@ -162,18 +162,18 @@ pub fn get_fork_choice_head( while { let current_block = blocks .get(&block_hash) - .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; + .ok_or_else(|| anyhow!("Block not found for vote head: {}", block_hash))?; let root_block = blocks .get(&root) - .ok_or_else(|| anyhow!("Root block not found for hash: {}", root))?; + .ok_or_else(|| anyhow!("Block not found for root: {}", root))?; current_block.slot > root_block.slot } { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); vote_weights.insert(block_hash, current_weights + 1); - let current_block = blocks + block_hash = blocks .get(&block_hash) - .ok_or_else(|| anyhow!("Block not found for hash: {}", block_hash))?; - block_hash = current_block.parent; + .map(|block| block.parent) + .ok_or_else(|| anyhow!("Block not found for block parent: {}", block_hash))?; } } } From aa09db64234db64975680599354cb967984d513f Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 15:30:32 +0700 Subject: [PATCH 45/47] fix: LeanState::new() return the instance directly --- crates/common/consensus/lean/src/state.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index a4b0a3552..3d93ac3c6 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -31,8 +31,8 @@ pub struct LeanState { } impl LeanState { - pub fn new(num_validators: u64) -> anyhow::Result { - Ok(LeanState { + pub fn new(num_validators: u64) -> LeanState { + LeanState { config: Config { num_validators }, latest_justified_hash: B256::ZERO, @@ -44,10 +44,9 @@ impl LeanState { justified_slots: VariableList::empty(), justifications_roots: VariableList::empty(), - justifications_roots_validators: BitList::with_capacity(0).map_err(|err| { - anyhow!("Failed to initialize state's justifications_roots_validators: {err:?}") - })?, - }) + justifications_roots_validators: BitList::with_capacity(0) + .expect("Failed to initialize an empty BitList"), + } } fn get_justifications_roots_index(&self, root: &B256) -> Option { @@ -177,7 +176,7 @@ mod test { #[test] fn initialize_justifications_for_root() { - let mut state = LeanState::new(1).unwrap(); + let mut state = LeanState::new(1); // Initialize 1st root state @@ -212,7 +211,7 @@ mod test { #[test] fn set_justification() { - let mut state = LeanState::new(1).unwrap(); + let mut state = LeanState::new(1); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); let validator_id = 7u64; @@ -244,7 +243,7 @@ mod test { #[test] fn count_justifications() { - let mut state = LeanState::new(1).unwrap(); + let mut state = LeanState::new(1); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); @@ -271,7 +270,7 @@ mod test { #[test] fn remove_justifications() { // Assuming 3 roots & 4 validators - let mut state = LeanState::new(3).unwrap(); + let mut state = LeanState::new(3); let root0 = B256::repeat_byte(1); let root1 = B256::repeat_byte(2); let root2 = B256::repeat_byte(3); From 5f32a912d859ef6b0ffc55f1387b150e60466d7d Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 15:32:39 +0700 Subject: [PATCH 46/47] refactor: return early for initialize_justifications... --- crates/common/consensus/lean/src/state.rs | 58 ++++++++++++----------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 3d93ac3c6..92aaf5984 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -54,35 +54,39 @@ impl LeanState { } pub fn initialize_justifications_for_root(&mut self, root: &B256) -> anyhow::Result<()> { - if !self.justifications_roots.contains(root) { - self.justifications_roots.push(*root).map_err(|err| { - anyhow!("Failed to insert root into justifications_roots: {err:?}") - })?; + // Return early if the justifications are already initialized + if self.justifications_roots.contains(root) { + return Ok(()); + } + + self.justifications_roots.push(*root).map_err(|err| { + anyhow!("Failed to insert root into justifications_roots: {err:?}") + })?; + + let old_length = self.justifications_roots_validators.len(); + let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; + + let mut new_justifications_roots_validators = BitList::with_capacity(new_length) + .map_err(|err| anyhow!("Failed to initialize new justification bits: {err:?}"))?; + + for (i, bit) in self.justifications_roots_validators.iter().enumerate() { + new_justifications_roots_validators + .set(i, bit) + .map_err(|err| { + anyhow!( + "Failed to initialize justification bits to existing values: {err:?}" + ) + })?; + } - let old_length = self.justifications_roots_validators.len(); - let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; - - let mut new_justifications_roots_validators = BitList::with_capacity(new_length) - .map_err(|err| anyhow!("Failed to initialize new justification bits: {err:?}"))?; - - for (i, bit) in self.justifications_roots_validators.iter().enumerate() { - new_justifications_roots_validators - .set(i, bit) - .map_err(|err| { - anyhow!( - "Failed to initialize justification bits to existing values: {err:?}" - ) - })?; - } - - for i in old_length..new_length { - new_justifications_roots_validators - .set(i, false) - .map_err(|err| anyhow!("Failed to zero-fill justification bits: {err:?}"))?; - } - - self.justifications_roots_validators = new_justifications_roots_validators; + for i in old_length..new_length { + new_justifications_roots_validators + .set(i, false) + .map_err(|err| anyhow!("Failed to zero-fill justification bits: {err:?}"))?; } + + self.justifications_roots_validators = new_justifications_roots_validators; + Ok(()) } From 3bbd246cd1a19e969cb644e8d12d92f3f96c19fa Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 5 Aug 2025 15:46:04 +0700 Subject: [PATCH 47/47] refactor: anyhow consistency --- crates/common/chain/lean/src/staker.rs | 28 +++++++++++------- crates/common/consensus/lean/src/lib.rs | 10 +++---- crates/common/consensus/lean/src/state.rs | 36 ++++++++++------------- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/crates/common/chain/lean/src/staker.rs b/crates/common/chain/lean/src/staker.rs index e0a84c078..07e278443 100644 --- a/crates/common/chain/lean/src/staker.rs +++ b/crates/common/chain/lean/src/staker.rs @@ -118,7 +118,7 @@ impl Staker { let network = self .network .lock() - .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; + .map_err(|err| anyhow::anyhow!("Failed to acquire network lock: {err:?}"))?; network.time % SLOT_DURATION }; @@ -151,7 +151,7 @@ impl Staker { let network = self .network .lock() - .map_err(|e| anyhow::anyhow!("Failed to acquire network lock: {}", e))?; + .map_err(|err| anyhow::anyhow!("Failed to acquire network lock: {err:?}"))?; Ok(network.time / SLOT_DURATION + 2) } @@ -162,7 +162,7 @@ impl Staker { let head_block = self .chain .get(&self.head) - .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; + .ok_or_else(|| anyhow::anyhow!("Block not found in chain for head: {}", self.head))?; info!( "proposing (Staker {}), head = {}", @@ -172,7 +172,7 @@ impl Staker { let head_state = self .post_states .get(&self.head) - .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; + .ok_or_else(|| anyhow::anyhow!("Post state not found for head: {}", self.head))?; let mut new_block = Block { slot: new_slot, parent: self.head, @@ -202,7 +202,7 @@ impl Staker { new_block .votes .push(vote) - .map_err(|e| anyhow::anyhow!("Failed to add vote to new_block: {:?}", e))?; + .map_err(|err| anyhow::anyhow!("Failed to add vote to new_block: {err:?}"))?; } } @@ -225,21 +225,24 @@ impl Staker { let state = self .post_states .get(&self.head) - .ok_or_else(|| anyhow::anyhow!("Head state not found for hash: {}", self.head))?; + .ok_or_else(|| anyhow::anyhow!("Post state not found for head: {}", self.head))?; let mut target_block = self .chain .get(&self.head) - .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; + .ok_or_else(|| anyhow::anyhow!("Block not found in chain for head: {}", self.head))?; // If there is no very recent safe target, then vote for the k'th ancestor // of the head for _ in 0..3 { let safe_target_block = self.chain.get(&self.safe_target).ok_or_else(|| { - anyhow::anyhow!("Safe target block not found for hash: {}", self.safe_target) + anyhow::anyhow!("Block not found for safe target hash: {}", self.safe_target) })?; if target_block.slot > safe_target_block.slot { target_block = self.chain.get(&target_block.parent).ok_or_else(|| { - anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent) + anyhow::anyhow!( + "Block not found for target block's parent hash: {}", + target_block.parent + ) })?; } } @@ -248,14 +251,17 @@ impl Staker { // valid to justify, make sure the target is one of those while !is_justifiable_slot(&state.latest_finalized_slot, &target_block.slot) { target_block = self.chain.get(&target_block.parent).ok_or_else(|| { - anyhow::anyhow!("Parent block not found for hash: {}", target_block.parent) + anyhow::anyhow!( + "Block not found for target block's parent hash: {}", + target_block.parent + ) })?; } let head_block = self .chain .get(&self.head) - .ok_or_else(|| anyhow::anyhow!("Head block not found for hash: {}", self.head))?; + .ok_or_else(|| anyhow::anyhow!("Block not found for head: {}", self.head))?; let vote = Vote { validator_id: self.validator_id, diff --git a/crates/common/consensus/lean/src/lib.rs b/crates/common/consensus/lean/src/lib.rs index df7cdd701..096b371c9 100644 --- a/crates/common/consensus/lean/src/lib.rs +++ b/crates/common/consensus/lean/src/lib.rs @@ -162,10 +162,10 @@ pub fn get_fork_choice_head( while { let current_block = blocks .get(&block_hash) - .ok_or_else(|| anyhow!("Block not found for vote head: {}", block_hash))?; + .ok_or_else(|| anyhow!("Block not found for vote head: {block_hash}"))?; let root_block = blocks .get(&root) - .ok_or_else(|| anyhow!("Block not found for root: {}", root))?; + .ok_or_else(|| anyhow!("Block not found for root: {root}"))?; current_block.slot > root_block.slot } { let current_weights = vote_weights.get(&block_hash).unwrap_or(&0); @@ -173,7 +173,7 @@ pub fn get_fork_choice_head( block_hash = blocks .get(&block_hash) .map(|block| block.parent) - .ok_or_else(|| anyhow!("Block not found for block parent: {}", block_hash))?; + .ok_or_else(|| anyhow!("Block not found for block parent: {block_hash}"))?; } } } @@ -206,9 +206,7 @@ pub fn get_fork_choice_head( let slot = blocks.get(*child_hash).map(|block| block.slot).unwrap_or(0); (*vote_weight, slot, *(*child_hash)) }) - .ok_or_else(|| { - anyhow!("No children found for current root: {}", current_root) - })?; + .ok_or_else(|| anyhow!("No children found for current root: {current_root}"))?; } } } diff --git a/crates/common/consensus/lean/src/state.rs b/crates/common/consensus/lean/src/state.rs index 92aaf5984..a1bfe7d25 100644 --- a/crates/common/consensus/lean/src/state.rs +++ b/crates/common/consensus/lean/src/state.rs @@ -54,14 +54,13 @@ impl LeanState { } pub fn initialize_justifications_for_root(&mut self, root: &B256) -> anyhow::Result<()> { - // Return early if the justifications are already initialized if self.justifications_roots.contains(root) { return Ok(()); } - self.justifications_roots.push(*root).map_err(|err| { - anyhow!("Failed to insert root into justifications_roots: {err:?}") - })?; + self.justifications_roots + .push(*root) + .map_err(|err| anyhow!("Failed to insert root into justifications_roots: {err:?}"))?; let old_length = self.justifications_roots_validators.len(); let new_length = old_length + VALIDATOR_REGISTRY_LIMIT as usize; @@ -73,9 +72,7 @@ impl LeanState { new_justifications_roots_validators .set(i, bit) .map_err(|err| { - anyhow!( - "Failed to initialize justification bits to existing values: {err:?}" - ) + anyhow!("Failed to initialize justification bits to existing values: {err:?}") })?; } @@ -96,9 +93,9 @@ impl LeanState { validator_id: &u64, value: bool, ) -> anyhow::Result<()> { - let index = self - .get_justifications_roots_index(root) - .ok_or(anyhow!("Failed to find the justifications index to set"))?; + let index = self.get_justifications_roots_index(root).ok_or_else(|| { + anyhow!("Failed to find the justifications index to set for root: {root}") + })?; self.justifications_roots_validators .set( @@ -113,7 +110,7 @@ impl LeanState { pub fn count_justifications(&self, root: &B256) -> anyhow::Result { let index = self .get_justifications_roots_index(root) - .ok_or_else(|| anyhow!("Could not find justifications for the provided block root"))?; + .ok_or_else(|| anyhow!("Could not find justifications for root: {root}"))?; let start_range = index * VALIDATOR_REGISTRY_LIMIT as usize; @@ -128,18 +125,15 @@ impl LeanState { } pub fn remove_justifications(&mut self, root: &B256) -> anyhow::Result<()> { - let index = self - .get_justifications_roots_index(root) - .ok_or_else(|| anyhow!("Failed to find the justifications index to remove"))?; + let index = self.get_justifications_roots_index(root).ok_or_else(|| { + anyhow!("Failed to find the justifications index to remove for root: {root}") + })?; self.justifications_roots.remove(index); let new_length = self.justifications_roots.len() * VALIDATOR_REGISTRY_LIMIT as usize; let mut new_justifications_roots_validators = - BitList::::with_capacity(new_length).map_err(|e| { - anyhow!( - "Failed to recreate state's justifications_roots_validators: {:?}", - e - ) + BitList::::with_capacity(new_length).map_err(|err| { + anyhow!("Failed to recreate state's justifications_roots_validators: {err:?}") })?; // Take left side of the list (if any) @@ -151,7 +145,7 @@ impl LeanState { { new_justifications_roots_validators .set(i, justification_bit) - .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; + .map_err(|err| anyhow!("Failed to set new justification bit: {err:?}"))?; } // Take right side of the list (if any) @@ -166,7 +160,7 @@ impl LeanState { index * VALIDATOR_REGISTRY_LIMIT as usize + i, justification_bit, ) - .map_err(|e| anyhow!("Failed to set new justification bit: {:?}", e))?; + .map_err(|err| anyhow!("Failed to set new justification bit: {err:?}"))?; } self.justifications_roots_validators = new_justifications_roots_validators;