From 6aaf90e60fdde0612a18dd04ca5a915ed01d35e2 Mon Sep 17 00:00:00 2001 From: debris Date: Sun, 18 Feb 2018 15:07:36 +0100 Subject: [PATCH 1/5] fixed broken logs --- ethcore/src/blockchain/block_info.rs | 4 +- ethcore/src/blockchain/blockchain.rs | 91 +++++++++++++++++++++----- ethcore/src/blockchain/generator.rs | 5 ++ ethcore/src/blooms/bloom_group.rs | 8 +++ ethcore/src/snapshot/consensus/work.rs | 4 +- util/bloomchain/src/group/group.rs | 2 +- 6 files changed, 93 insertions(+), 21 deletions(-) diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs index ade08c7d233..ee8a50d09d9 100644 --- a/ethcore/src/blockchain/block_info.rs +++ b/ethcore/src/blockchain/block_info.rs @@ -31,7 +31,7 @@ pub struct BlockInfo { } /// Describes location of newly inserted block. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub enum BlockLocation { /// It's part of the canon chain. CanonChain, @@ -43,7 +43,7 @@ pub enum BlockLocation { BranchBecomingCanonChain(BranchBecomingCanonChainData), } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct BranchBecomingCanonChainData { /// Hash of the newest common ancestor with old canon chain. pub ancestor: H256, diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index feedf1247c6..421a8a13d4a 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -16,7 +16,7 @@ //! Blockchain database. -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, hash_map}; use std::sync::Arc; use std::mem; use itertools::Itertools; @@ -714,6 +714,7 @@ impl BlockChain { /// This is used by snapshot restoration and when downloading missing blocks for the chain gap. /// `is_best` forces the best block to be updated to this block. /// `is_ancient` forces the best block of the first block sequence to be updated to this block. + /// `parent_td` is a parent total diffuculty /// Supply a dummy parent total difficulty when the parent block may not be in the chain. /// Returns true if the block is disconnected. pub fn insert_unordered_block(&self, batch: &mut DBTransaction, bytes: &[u8], receipts: Vec, parent_td: Option, is_best: bool, is_ancient: bool) -> bool { @@ -1005,29 +1006,50 @@ impl BlockChain { batch.extend_with_cache(db::COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); } - { - let mut write_blocks_blooms = self.blocks_blooms.write(); - batch.extend_with_cache(db::COL_EXTRA, &mut *write_blocks_blooms, update.blocks_blooms, CacheUpdatePolicy::Remove); - } - // These cached values must be updated last with all four locks taken to avoid // cache decoherence { let mut best_block = self.pending_best_block.write(); + let mut write_blocks_blooms = self.blocks_blooms.write(); // update best block match update.info.location { BlockLocation::Branch => (), - _ => if is_best { - batch.put(db::COL_EXTRA, b"best", &update.info.hash); - *best_block = Some(BestBlock { - hash: update.info.hash, - number: update.info.number, - total_difficulty: update.info.total_difficulty, - timestamp: update.timestamp, - block: update.block.to_vec(), - }); + BlockLocation::BranchBecomingCanonChain(_) => { + // clear all existing blooms, cause they may be created for block + // number higher than current best block + *write_blocks_blooms = update.blocks_blooms; + for (key, value) in write_blocks_blooms.iter() { + batch.write(db::COL_EXTRA, key, value); + } + }, + BlockLocation::CanonChain => { + // update all existing blooms groups + for (key, value) in update.blocks_blooms { + match write_blocks_blooms.entry(key) { + hash_map::Entry::Occupied(mut entry) => { + entry.get_mut().accrue_bloom_group(&value); + batch.write(db::COL_EXTRA, entry.key(), entry.get()); + }, + hash_map::Entry::Vacant(entry) => { + batch.write(db::COL_EXTRA, entry.key(), &value); + entry.insert(value); + }, + } + } }, } + + if is_best && update.info.location != BlockLocation::Branch { + batch.put(db::COL_EXTRA, b"best", &update.info.hash); + *best_block = Some(BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty, + timestamp: update.timestamp, + block: update.block.to_vec(), + }); + } + let mut write_hashes = self.pending_block_hashes.write(); let mut write_details = self.pending_block_details.write(); let mut write_txs = self.pending_transaction_addresses.write(); @@ -2098,6 +2120,45 @@ mod tests { assert_eq!(blocks_ba, vec![3]); } + #[test] + fn test_insert_unordered() { + let bloom_b1: Bloom = "00000020000000000000000000000000000000000000000002000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000400000000000000000000002000".into(); + + let bloom_b2: Bloom = "00000000000000000000000000000000000000000000020000001000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000".into(); + + let bloom_b3: Bloom = "00000000000000000000000000000000000000000000020000000800000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000008000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000002000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000000000".into(); + + let genesis = BlockBuilder::genesis(); + let b1 = genesis.add_block_with_bloom(bloom_b1); + let b2 = b1.add_block_with_bloom(bloom_b2); + let b3 = b2.add_block_with_bloom(bloom_b3); + let b1_total_difficulty = genesis.last().difficulty() + b1.last().difficulty(); + + let db = new_db(); + let bc = new_chain(&genesis.last().encoded(), db.clone()); + let mut batch = db.transaction(); + bc.insert_unordered_block(&mut batch, &b2.last().encoded(), vec![], Some(b1_total_difficulty), false, false); + bc.commit(); + bc.insert_unordered_block(&mut batch, &b3.last().encoded(), vec![], None, true, false); + bc.commit(); + bc.insert_unordered_block(&mut batch, &b1.last().encoded(), vec![], None, false, false); + bc.commit(); + db.write(batch).unwrap(); + + assert_eq!(bc.best_block_hash(), b3.last().hash()); + assert_eq!(bc.block_hash(1).unwrap(), b1.last().hash()); + assert_eq!(bc.block_hash(2).unwrap(), b2.last().hash()); + assert_eq!(bc.block_hash(3).unwrap(), b3.last().hash()); + + let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 3); + let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 3); + let blocks_b3 = bc.blocks_with_bloom(&bloom_b3, 0, 3); + + assert_eq!(blocks_b1, vec![1]); + assert_eq!(blocks_b2, vec![2]); + assert_eq!(blocks_b3, vec![3]); + } + #[test] fn test_best_block_update() { let genesis = BlockBuilder::genesis(); diff --git a/ethcore/src/blockchain/generator.rs b/ethcore/src/blockchain/generator.rs index 5c9068721c8..5f990a4d980 100644 --- a/ethcore/src/blockchain/generator.rs +++ b/ethcore/src/blockchain/generator.rs @@ -53,6 +53,11 @@ impl Block { pub fn encoded(&self) -> Bytes { encode(self).into_vec() } + + #[inline] + pub fn difficulty(&self) -> U256 { + *self.header.difficulty() + } } #[derive(Debug)] diff --git a/ethcore/src/blooms/bloom_group.rs b/ethcore/src/blooms/bloom_group.rs index 4cd891686cb..4d57c9b2622 100644 --- a/ethcore/src/blooms/bloom_group.rs +++ b/ethcore/src/blooms/bloom_group.rs @@ -24,6 +24,14 @@ pub struct BloomGroup { blooms: Vec, } +impl BloomGroup { + pub fn accrue_bloom_group(&mut self, group: &BloomGroup) { + for (bloom, other) in self.blooms.iter_mut().zip(group.blooms.iter()) { + bloom.accrue_bloom(other); + } + } +} + impl From for BloomGroup { fn from(group: bc::BloomGroup) -> Self { let blooms = group.blooms diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/src/snapshot/consensus/work.rs index bfb213091e5..5b4ff488893 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/src/snapshot/consensus/work.rs @@ -248,9 +248,7 @@ impl Rebuilder for PowRebuilder { let abridged_rlp = pair.at(0)?.as_raw().to_owned(); let abridged_block = AbridgedBlock::from_raw(abridged_rlp); let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?; - let receipts_root = ordered_trie_root( - pair.at(1)?.iter().map(|r| r.as_raw()) - ); + let receipts_root = ordered_trie_root(pair.at(1)?.iter().map(|r| r.as_raw())); let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; let block_bytes = block.rlp_bytes(Seal::With); diff --git a/util/bloomchain/src/group/group.rs b/util/bloomchain/src/group/group.rs index 16dbea65e3d..084c8f8e48e 100644 --- a/util/bloomchain/src/group/group.rs +++ b/util/bloomchain/src/group/group.rs @@ -1,7 +1,7 @@ use bloom::Bloom; /// Group of blooms that are in the same index. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct BloomGroup { pub blooms: Vec, } From 8b43f61a65cf9de10faaa041fb5773a4dbb00101 Mon Sep 17 00:00:00 2001 From: debris Date: Mon, 19 Feb 2018 16:09:17 +0100 Subject: [PATCH 2/5] bring back old lock order --- ethcore/src/blockchain/blockchain.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 421a8a13d4a..e72d6d2cf51 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -1006,10 +1006,7 @@ impl BlockChain { batch.extend_with_cache(db::COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); } - // These cached values must be updated last with all four locks taken to avoid - // cache decoherence { - let mut best_block = self.pending_best_block.write(); let mut write_blocks_blooms = self.blocks_blooms.write(); // update best block match update.info.location { @@ -1038,7 +1035,12 @@ impl BlockChain { } }, } + } + // These cached values must be updated last with all four locks taken to avoid + // cache decoherence + { + let mut best_block = self.pending_best_block.write(); if is_best && update.info.location != BlockLocation::Branch { batch.put(db::COL_EXTRA, b"best", &update.info.hash); *best_block = Some(BestBlock { From 1fa4de98c6551b73beaf0e4021d9f2dd44072566 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 21 Feb 2018 11:29:29 +0100 Subject: [PATCH 3/5] removed bloom groups from blockchain --- ethcore/src/blockchain/blockchain.rs | 127 ++------------------------- ethcore/src/blockchain/cache.rs | 4 +- ethcore/src/blockchain/extras.rs | 28 ------ ethcore/src/blockchain/update.rs | 3 - ethcore/src/blooms/bloom_group.rs | 8 -- 5 files changed, 8 insertions(+), 162 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 64b6e936dfd..aeb50ed1807 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -16,11 +16,10 @@ //! Blockchain database. -use std::collections::{HashMap, HashSet, hash_map}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::mem; use itertools::Itertools; -use bloomchain as bc; use heapsize::HeapSizeOf; use ethereum_types::{H256, Bloom, U256}; use parking_lot::{Mutex, RwLock}; @@ -31,7 +30,6 @@ use transaction::*; use views::*; use log_entry::{LogEntry, LocalizedLogEntry}; use receipt::Receipt; -use blooms::{BloomGroup, GroupPosition}; use blockchain::best_block::{BestBlock, BestAncientBlock}; use blockchain::block_info::{BlockInfo, BlockLocation, BranchBecomingCanonChainData}; use blockchain::extras::{BlockReceipts, BlockDetails, TransactionAddress, EPOCH_KEY_PREFIX, EpochTransitions}; @@ -47,9 +45,6 @@ use rayon::prelude::*; use ansi_term::Colour; use kvdb::{DBTransaction, KeyValueDB}; -const LOG_BLOOMS_LEVELS: usize = 3; -const LOG_BLOOMS_ELEMENTS_PER_INDEX: usize = 16; - /// Interface for querying blocks by hash and by number. pub trait BlockProvider { /// Returns true if the given block is known @@ -163,26 +158,14 @@ enum CacheId { BlockDetails(H256), BlockHashes(BlockNumber), TransactionAddresses(H256), - BlocksBlooms(GroupPosition), BlockReceipts(H256), } -impl bc::group::BloomGroupDatabase for BlockChain { - fn blooms_at(&self, position: &bc::group::GroupPosition) -> Option { - let position = GroupPosition::from(position.clone()); - let result = self.db.read_with_cache(db::COL_EXTRA, &self.blocks_blooms, &position).map(Into::into); - self.cache_man.lock().note_used(CacheId::BlocksBlooms(position)); - result - } -} - /// Structure providing fast access to blockchain data. /// /// **Does not do input data verification.** pub struct BlockChain { // All locks must be captured in the order declared here. - blooms_config: bc::Config, - best_block: RwLock, // Stores best block of the first uninterrupted sequence of blocks. `None` if there are no gaps. // Only updated with `insert_unordered_block`. @@ -199,7 +182,6 @@ pub struct BlockChain { block_details: RwLock>, block_hashes: RwLock>, transaction_addresses: RwLock>, - blocks_blooms: RwLock>, block_receipts: RwLock>, db: Arc, @@ -351,11 +333,13 @@ impl BlockProvider for BlockChain { /// Returns numbers of blocks containing given bloom. fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec { - let range = from_block as bc::Number..to_block as bc::Number; - let chain = bc::group::BloomGroupChain::new(self.blooms_config, self); - chain.with_bloom(&range, bloom) + // +1, cause it's inclusive range + (from_block..to_block + 1) .into_iter() - .map(|b| b as BlockNumber) + .filter_map(|number| self.block_hash(number).map(|hash| (number, hash))) + .map(|(number, hash)| (number, self.block_header_data(&hash).expect("hash exists; qed"))) + .filter(|&(_, ref header)| header.view().log_bloom().contains(bloom)) + .map(|(number, _)| number) .collect() } @@ -487,10 +471,6 @@ impl BlockChain { let cache_man = CacheManager::new(config.pref_cache_size, config.max_cache_size, 400); let mut bc = BlockChain { - blooms_config: bc::Config { - levels: LOG_BLOOMS_LEVELS, - elements_per_index: LOG_BLOOMS_ELEMENTS_PER_INDEX, - }, first_block: None, best_block: RwLock::new(BestBlock::default()), best_ancient_block: RwLock::new(None), @@ -499,7 +479,6 @@ impl BlockChain { block_details: RwLock::new(HashMap::new()), block_hashes: RwLock::new(HashMap::new()), transaction_addresses: RwLock::new(HashMap::new()), - blocks_blooms: RwLock::new(HashMap::new()), block_receipts: RwLock::new(HashMap::new()), db: db.clone(), cache_man: Mutex::new(cache_man), @@ -751,7 +730,6 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: self.prepare_block_details_update(bytes, &info), block_receipts: self.prepare_block_receipts_update(receipts, &info), - blocks_blooms: self.prepare_block_blooms_update(bytes, &info), transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info, timestamp: header.timestamp(), @@ -800,7 +778,6 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: update, block_receipts: self.prepare_block_receipts_update(receipts, &info), - blocks_blooms: self.prepare_block_blooms_update(bytes, &info), transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info, timestamp: header.timestamp(), @@ -948,7 +925,6 @@ impl BlockChain { block_hashes: self.prepare_block_hashes_update(bytes, &info), block_details: self.prepare_block_details_update(bytes, &info), block_receipts: self.prepare_block_receipts_update(receipts, &info), - blocks_blooms: self.prepare_block_blooms_update(bytes, &info), transactions_addresses: self.prepare_transaction_addresses_update(bytes, &info), info: info.clone(), timestamp: header.timestamp(), @@ -1006,37 +982,6 @@ impl BlockChain { batch.extend_with_cache(db::COL_EXTRA, &mut *write_receipts, update.block_receipts, CacheUpdatePolicy::Remove); } - { - let mut write_blocks_blooms = self.blocks_blooms.write(); - // update best block - match update.info.location { - BlockLocation::Branch => (), - BlockLocation::BranchBecomingCanonChain(_) => { - // clear all existing blooms, cause they may be created for block - // number higher than current best block - *write_blocks_blooms = update.blocks_blooms; - for (key, value) in write_blocks_blooms.iter() { - batch.write(db::COL_EXTRA, key, value); - } - }, - BlockLocation::CanonChain => { - // update all existing blooms groups - for (key, value) in update.blocks_blooms { - match write_blocks_blooms.entry(key) { - hash_map::Entry::Occupied(mut entry) => { - entry.get_mut().accrue_bloom_group(&value); - batch.write(db::COL_EXTRA, entry.key(), entry.get()); - }, - hash_map::Entry::Vacant(entry) => { - batch.write(db::COL_EXTRA, entry.key(), &value); - entry.insert(value); - }, - } - } - }, - } - } - // These cached values must be updated last with all four locks taken to avoid // cache decoherence { @@ -1268,59 +1213,6 @@ impl BlockChain { } } - /// This functions returns modified blocks blooms. - /// - /// To accelerate blooms lookups, blomms are stored in multiple - /// layers (BLOOM_LEVELS, currently 3). - /// ChainFilter is responsible for building and rebuilding these layers. - /// It returns them in HashMap, where values are Blooms and - /// keys are BloomIndexes. BloomIndex represents bloom location on one - /// of these layers. - /// - /// To reduce number of queries to databse, block blooms are stored - /// in BlocksBlooms structure which contains info about several - /// (BLOOM_INDEX_SIZE, currently 16) consecutive blocks blooms. - /// - /// Later, BloomIndexer is used to map bloom location on filter layer (BloomIndex) - /// to bloom location in database (BlocksBloomLocation). - /// - fn prepare_block_blooms_update(&self, block_bytes: &[u8], info: &BlockInfo) -> HashMap { - let block = BlockView::new(block_bytes); - let header = block.header_view(); - - let log_blooms = match info.location { - BlockLocation::Branch => HashMap::new(), - BlockLocation::CanonChain => { - let log_bloom = header.log_bloom(); - if log_bloom.is_zero() { - HashMap::new() - } else { - let chain = bc::group::BloomGroupChain::new(self.blooms_config, self); - chain.insert(info.number as bc::Number, log_bloom) - } - }, - BlockLocation::BranchBecomingCanonChain(ref data) => { - let ancestor_number = self.block_number(&data.ancestor).unwrap(); - let start_number = ancestor_number + 1; - let range = start_number as bc::Number..self.best_block_number() as bc::Number; - - let mut blooms: Vec = data.enacted.iter() - .map(|hash| self.block_header_data(hash).unwrap()) - .map(|h| h.log_bloom()) - .collect(); - - blooms.push(header.log_bloom()); - - let chain = bc::group::BloomGroupChain::new(self.blooms_config, self); - chain.replace(&range, blooms) - } - }; - - log_blooms.into_iter() - .map(|p| (From::from(p.0), From::from(p.1))) - .collect() - } - /// Get best block hash. pub fn best_block_hash(&self) -> H256 { self.best_block.read().hash @@ -1354,7 +1246,6 @@ impl BlockChain { blocks: self.block_headers.read().heap_size_of_children() + self.block_bodies.read().heap_size_of_children(), block_details: self.block_details.read().heap_size_of_children(), transaction_addresses: self.transaction_addresses.read().heap_size_of_children(), - blocks_blooms: self.blocks_blooms.read().heap_size_of_children(), block_receipts: self.block_receipts.read().heap_size_of_children(), } } @@ -1368,7 +1259,6 @@ impl BlockChain { let mut block_details = self.block_details.write(); let mut block_hashes = self.block_hashes.write(); let mut transaction_addresses = self.transaction_addresses.write(); - let mut blocks_blooms = self.blocks_blooms.write(); let mut block_receipts = self.block_receipts.write(); let mut cache_man = self.cache_man.lock(); @@ -1380,7 +1270,6 @@ impl BlockChain { CacheId::BlockDetails(ref h) => { block_details.remove(h); } CacheId::BlockHashes(ref h) => { block_hashes.remove(h); } CacheId::TransactionAddresses(ref h) => { transaction_addresses.remove(h); } - CacheId::BlocksBlooms(ref h) => { blocks_blooms.remove(h); } CacheId::BlockReceipts(ref h) => { block_receipts.remove(h); } } } @@ -1390,7 +1279,6 @@ impl BlockChain { block_details.shrink_to_fit(); block_hashes.shrink_to_fit(); transaction_addresses.shrink_to_fit(); - blocks_blooms.shrink_to_fit(); block_receipts.shrink_to_fit(); block_headers.heap_size_of_children() + @@ -1398,7 +1286,6 @@ impl BlockChain { block_details.heap_size_of_children() + block_hashes.heap_size_of_children() + transaction_addresses.heap_size_of_children() + - blocks_blooms.heap_size_of_children() + block_receipts.heap_size_of_children() }); } diff --git a/ethcore/src/blockchain/cache.rs b/ethcore/src/blockchain/cache.rs index 999be423df7..5a609f9d728 100644 --- a/ethcore/src/blockchain/cache.rs +++ b/ethcore/src/blockchain/cache.rs @@ -23,8 +23,6 @@ pub struct CacheSize { pub block_details: usize, /// Transaction addresses cache size. pub transaction_addresses: usize, - /// Blooms cache size. - pub blocks_blooms: usize, /// Block receipts size. pub block_receipts: usize, } @@ -32,6 +30,6 @@ pub struct CacheSize { impl CacheSize { /// Total amount used by the cache. pub fn total(&self) -> usize { - self.blocks + self.block_details + self.transaction_addresses + self.blocks_blooms + self.block_receipts + self.blocks + self.block_details + self.transaction_addresses + self.block_receipts } } diff --git a/ethcore/src/blockchain/extras.rs b/ethcore/src/blockchain/extras.rs index 97583a693a6..34a39f4aed6 100644 --- a/ethcore/src/blockchain/extras.rs +++ b/ethcore/src/blockchain/extras.rs @@ -18,7 +18,6 @@ use std::ops; use std::io::Write; -use blooms::{GroupPosition, BloomGroup}; use db::Key; use engines::epoch::{Transition as EpochTransition}; use header::BlockNumber; @@ -37,8 +36,6 @@ pub enum ExtrasIndex { BlockHash = 1, /// Transaction address index TransactionAddress = 2, - /// Block blooms index - BlocksBlooms = 3, /// Block receipts index BlockReceipts = 4, /// Epoch transition data index. @@ -86,31 +83,6 @@ impl Key for H256 { } } -pub struct LogGroupKey([u8; 6]); - -impl ops::Deref for LogGroupKey { - type Target = [u8]; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Key for GroupPosition { - type Target = LogGroupKey; - - fn key(&self) -> Self::Target { - let mut result = [0u8; 6]; - result[0] = ExtrasIndex::BlocksBlooms as u8; - result[1] = self.level; - result[2] = (self.index >> 24) as u8; - result[3] = (self.index >> 16) as u8; - result[4] = (self.index >> 8) as u8; - result[5] = self.index as u8; - LogGroupKey(result) - } -} - impl Key for H256 { type Target = H264; diff --git a/ethcore/src/blockchain/update.rs b/ethcore/src/blockchain/update.rs index 467ccc8d17b..54fc0659718 100644 --- a/ethcore/src/blockchain/update.rs +++ b/ethcore/src/blockchain/update.rs @@ -3,7 +3,6 @@ use ethereum_types::H256; use header::BlockNumber; use blockchain::block_info::BlockInfo; use blockchain::extras::{BlockDetails, BlockReceipts, TransactionAddress}; -use blooms::{BloomGroup, GroupPosition}; /// Block extras update info. pub struct ExtrasUpdate<'a> { @@ -19,8 +18,6 @@ pub struct ExtrasUpdate<'a> { pub block_details: HashMap, /// Modified block receipts. pub block_receipts: HashMap, - /// Modified blocks blooms. - pub blocks_blooms: HashMap, /// Modified transaction addresses (None signifies removed transactions). pub transactions_addresses: HashMap>, } diff --git a/ethcore/src/blooms/bloom_group.rs b/ethcore/src/blooms/bloom_group.rs index 4b47b1ad947..bb4a160b8a0 100644 --- a/ethcore/src/blooms/bloom_group.rs +++ b/ethcore/src/blooms/bloom_group.rs @@ -24,14 +24,6 @@ pub struct BloomGroup { blooms: Vec, } -impl BloomGroup { - pub fn accrue_bloom_group(&mut self, group: &BloomGroup) { - for (bloom, other) in self.blooms.iter_mut().zip(group.blooms.iter()) { - bloom.accrue_bloom(other); - } - } -} - impl From for BloomGroup { fn from(group: bc::BloomGroup) -> Self { BloomGroup { From 53f399d2f3ece9a44a17554a24bc8da656949585 Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 21 Feb 2018 11:38:22 +0100 Subject: [PATCH 4/5] revert unrelated changes --- ethcore/src/blockchain/block_info.rs | 4 ++-- ethcore/src/blockchain/blockchain.rs | 21 ++++++++++++--------- ethcore/src/snapshot/consensus/work.rs | 4 +++- util/bloomchain/src/group/group.rs | 2 +- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ethcore/src/blockchain/block_info.rs b/ethcore/src/blockchain/block_info.rs index ee8a50d09d9..ade08c7d233 100644 --- a/ethcore/src/blockchain/block_info.rs +++ b/ethcore/src/blockchain/block_info.rs @@ -31,7 +31,7 @@ pub struct BlockInfo { } /// Describes location of newly inserted block. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub enum BlockLocation { /// It's part of the canon chain. CanonChain, @@ -43,7 +43,7 @@ pub enum BlockLocation { BranchBecomingCanonChain(BranchBecomingCanonChainData), } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone)] pub struct BranchBecomingCanonChainData { /// Hash of the newest common ancestor with old canon chain. pub ancestor: H256, diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index aeb50ed1807..fd23458c6de 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -986,15 +986,18 @@ impl BlockChain { // cache decoherence { let mut best_block = self.pending_best_block.write(); - if is_best && update.info.location != BlockLocation::Branch { - batch.put(db::COL_EXTRA, b"best", &update.info.hash); - *best_block = Some(BestBlock { - hash: update.info.hash, - number: update.info.number, - total_difficulty: update.info.total_difficulty, - timestamp: update.timestamp, - block: update.block.to_vec(), - }); + match update.info.location { + BlockLocation::Branch => (), + _ => if is_best { + batch.put(db::COL_EXTRA, b"best", &update.info.hash); + *best_block = Some(BestBlock { + hash: update.info.hash, + number: update.info.number, + total_difficulty: update.info.total_difficulty, + timestamp: update.timestamp, + block: update.block.to_vec(), + }); + } } let mut write_hashes = self.pending_block_hashes.write(); diff --git a/ethcore/src/snapshot/consensus/work.rs b/ethcore/src/snapshot/consensus/work.rs index 5b4ff488893..bfb213091e5 100644 --- a/ethcore/src/snapshot/consensus/work.rs +++ b/ethcore/src/snapshot/consensus/work.rs @@ -248,7 +248,9 @@ impl Rebuilder for PowRebuilder { let abridged_rlp = pair.at(0)?.as_raw().to_owned(); let abridged_block = AbridgedBlock::from_raw(abridged_rlp); let receipts: Vec<::receipt::Receipt> = pair.list_at(1)?; - let receipts_root = ordered_trie_root(pair.at(1)?.iter().map(|r| r.as_raw())); + let receipts_root = ordered_trie_root( + pair.at(1)?.iter().map(|r| r.as_raw()) + ); let block = abridged_block.to_block(parent_hash, cur_number, receipts_root)?; let block_bytes = block.rlp_bytes(Seal::With); diff --git a/util/bloomchain/src/group/group.rs b/util/bloomchain/src/group/group.rs index 084c8f8e48e..16dbea65e3d 100644 --- a/util/bloomchain/src/group/group.rs +++ b/util/bloomchain/src/group/group.rs @@ -1,7 +1,7 @@ use bloom::Bloom; /// Group of blooms that are in the same index. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct BloomGroup { pub blooms: Vec, } From b59073b8bc01d365335239ee0ea1b84ed5f80a2e Mon Sep 17 00:00:00 2001 From: debris Date: Wed, 21 Feb 2018 17:04:41 +0100 Subject: [PATCH 5/5] simplify blockchain_block_blooms --- ethcore/src/blockchain/blockchain.rs | 57 +++++++++++------------- ethcore/src/client/client.rs | 13 +----- ethcore/src/verification/verification.rs | 2 +- 3 files changed, 29 insertions(+), 43 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index fd23458c6de..bfe88b3fc24 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -144,7 +144,7 @@ pub trait BlockProvider { } /// Returns numbers of blocks containing given bloom. - fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec; + fn blocks_with_blooms(&self, bloom: &[Bloom], from_block: BlockNumber, to_block: BlockNumber) -> Vec; /// Returns logs matching given filter. fn logs(&self, blocks: Vec, matches: F, limit: Option) -> Vec @@ -331,14 +331,13 @@ impl BlockProvider for BlockChain { result } - /// Returns numbers of blocks containing given bloom. - fn blocks_with_bloom(&self, bloom: &Bloom, from_block: BlockNumber, to_block: BlockNumber) -> Vec { + fn blocks_with_blooms(&self, blooms: &[Bloom], from_block: BlockNumber, to_block: BlockNumber) -> Vec { // +1, cause it's inclusive range (from_block..to_block + 1) - .into_iter() + .into_par_iter() .filter_map(|number| self.block_hash(number).map(|hash| (number, hash))) .map(|(number, hash)| (number, self.block_header_data(&hash).expect("hash exists; qed"))) - .filter(|&(_, ref header)| header.view().log_bloom().contains(bloom)) + .filter(|&(_, ref header)| blooms.iter().any(|bloom| header.view().log_bloom().contains_bloom(bloom))) .map(|(number, _)| number) .collect() } @@ -355,19 +354,15 @@ impl BlockProvider for BlockChain { .filter_map(|number| self.block_hash(*number).map(|hash| (*number, hash))) .filter_map(|(number, hash)| self.block_receipts(&hash).map(|r| (number, hash, r.receipts))) .filter_map(|(number, hash, receipts)| self.block_body(&hash).map(|ref b| (number, hash, receipts, b.transaction_hashes()))) - .flat_map(|(number, hash, mut receipts, mut hashes)| { - if receipts.len() != hashes.len() { - warn!("Block {} ({}) has different number of receipts ({}) to transactions ({}). Database corrupt?", number, hash, receipts.len(), hashes.len()); - assert!(false); - } - let mut log_index = receipts.iter().fold(0, |sum, receipt| sum + receipt.logs.len()); + .flat_map(|(number, hash, receipts, hashes)| { + assert_eq!(receipts.len(), hashes.len(), "Block {} ({}) has different number of receipts ({}) to transactions ({})", number, hash, receipts.len(), hashes.len()); + let mut log_index: usize = receipts.iter().map(|r| r.logs.len()).sum(); let receipts_len = receipts.len(); - hashes.reverse(); - receipts.reverse(); receipts.into_iter() .map(|receipt| receipt.logs) .zip(hashes) + .rev() .enumerate() .flat_map(move |(index, (mut logs, tx_hash))| { let current_log_index = log_index; @@ -1965,46 +1960,46 @@ mod tests { let db = new_db(); let bc = new_chain(&genesis.last().encoded(), db.clone()); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); assert!(blocks_b1.is_empty()); assert!(blocks_b2.is_empty()); insert_block(&db, &bc, &b1.last().encoded(), vec![]); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); assert_eq!(blocks_b1, vec![1]); assert!(blocks_b2.is_empty()); insert_block(&db, &bc, &b2.last().encoded(), vec![]); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![2]); // hasn't been forked yet insert_block(&db, &bc, &b1a.last().encoded(), vec![]); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); - let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); + let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![2]); assert!(blocks_ba.is_empty()); // fork has happend insert_block(&db, &bc, &b2a.last().encoded(), vec![]); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); - let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); + let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5); assert!(blocks_b1.is_empty()); assert!(blocks_b2.is_empty()); assert_eq!(blocks_ba, vec![1, 2]); // fork back insert_block(&db, &bc, &b3.last().encoded(), vec![]); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 5); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 5); - let blocks_ba = bc.blocks_with_bloom(&bloom_ba, 0, 5); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 5); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 5); + let blocks_ba = bc.blocks_with_blooms(&[bloom_ba], 0, 5); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![2]); assert_eq!(blocks_ba, vec![3]); @@ -2040,9 +2035,9 @@ mod tests { assert_eq!(bc.block_hash(2).unwrap(), b2.last().hash()); assert_eq!(bc.block_hash(3).unwrap(), b3.last().hash()); - let blocks_b1 = bc.blocks_with_bloom(&bloom_b1, 0, 3); - let blocks_b2 = bc.blocks_with_bloom(&bloom_b2, 0, 3); - let blocks_b3 = bc.blocks_with_bloom(&bloom_b3, 0, 3); + let blocks_b1 = bc.blocks_with_blooms(&[bloom_b1], 0, 3); + let blocks_b2 = bc.blocks_with_blooms(&[bloom_b2], 0, 3); + let blocks_b3 = bc.blocks_with_blooms(&[bloom_b3], 0, 3); assert_eq!(blocks_b1, vec![1]); assert_eq!(blocks_b2, vec![2]); diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 0245238cefe..26dc538fe80 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -1677,17 +1677,8 @@ impl BlockChainClient for Client { }; let chain = self.chain.read(); - let blocks = filter.bloom_possibilities().iter() - .map(move |bloom| { - chain.blocks_with_bloom(bloom, from, to) - }) - .flat_map(|m| m) - // remove duplicate elements - .collect::>() - .into_iter() - .collect::>(); - - self.chain.read().logs(blocks, |entry| filter.matches(entry), filter.limit) + let blocks = chain.blocks_with_blooms(&filter.bloom_possibilities(), from, to); + chain.logs(blocks, |entry| filter.matches(entry), filter.limit) } fn filter_traces(&self, filter: TraceFilter) -> Option> { diff --git a/ethcore/src/verification/verification.rs b/ethcore/src/verification/verification.rs index 5e3bc7ea2ba..5b09c201f84 100644 --- a/ethcore/src/verification/verification.rs +++ b/ethcore/src/verification/verification.rs @@ -454,7 +454,7 @@ mod tests { unimplemented!() } - fn blocks_with_bloom(&self, _bloom: &Bloom, _from_block: BlockNumber, _to_block: BlockNumber) -> Vec { + fn blocks_with_blooms(&self, _blooms: &[Bloom], _from_block: BlockNumber, _to_block: BlockNumber) -> Vec { unimplemented!() }