diff --git a/ethcore/src/migrations/mod.rs b/ethcore/src/migrations/mod.rs index 7ccafac7434..7b9fb577695 100644 --- a/ethcore/src/migrations/mod.rs +++ b/ethcore/src/migrations/mod.rs @@ -26,3 +26,6 @@ pub use self::v9::Extract; mod v10; pub use self::v10::ToV10; + +mod upgrade_bloom; +pub use self::upgrade_bloom::upgrade_account_bloom; diff --git a/ethcore/src/migrations/upgrade_bloom.rs b/ethcore/src/migrations/upgrade_bloom.rs new file mode 100644 index 00000000000..6084d2c860c --- /dev/null +++ b/ethcore/src/migrations/upgrade_bloom.rs @@ -0,0 +1,137 @@ +// Copyright 2015, 2016 Ethcore (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +//! Bloom upgrade + +use db::{COL_EXTRA, COL_HEADERS, COL_STATE, NUM_COLUMNS, COL_ACCOUNT_BLOOM}; +use state_db::{ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET, StateDB, ACCOUNT_BLOOM_HASHCOUNT_KEY, ACCOUNT_BLOOM_SPACE_KEY}; +use util::trie::TrieDB; +use views::HeaderView; +use bloom_journal::Bloom; +use util::migration::Error; +use util::{journaldb, Trie}; +use util::{H256, FixedHash}; +use util::{Database, DatabaseConfig, DBTransaction, CompactionProfile}; +use std::path::Path; +use std::collections::HashMap; +use byteorder::{LittleEndian, ByteOrder}; + +fn check_bloom_exists(db: &Database) -> bool { + let hash_count_entry = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_HASHCOUNT_KEY) + .expect("Low-level database error"); + + hash_count_entry.is_some() +} + +fn check_space_match(db: &Database) -> Result<(), usize> { + let db_space = db.get(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_SPACE_KEY) + .expect("Low-level database error") + .map(|val| LittleEndian::read_u64(&val[..]) as usize) + // this was the initial size of the bloom which was not written in the database + .unwrap_or(1048576); + + if db_space == ACCOUNT_BLOOM_SPACE { Ok(()) } else { Err(db_space) } +} + +/// Account bloom upgrade routine. If bloom already present, does nothing. +/// If database empty (no best block), does nothing. +/// Can be called on upgraded database with no issues (will do nothing). +pub fn upgrade_account_bloom(db_path: &Path) -> Result<(), Error> { + let mut progress = ::util::migration::Progress::default(); + + let path = try!(db_path.to_str().ok_or(Error::MigrationImpossible)); + trace!(target: "migration", "Account bloom upgrade at {:?}", db_path); + + let source = try!(Database::open(&DatabaseConfig { + max_open_files: 64, + cache_sizes: HashMap::new(), + compaction: CompactionProfile::default(), + columns: NUM_COLUMNS, + wal: true, + }, path)); + + let best_block_hash = match try!(source.get(COL_EXTRA, b"best")) { + // no migration needed + None => { + trace!(target: "migration", "No best block hash, skipping"); + return Ok(()); + }, + Some(hash) => hash, + }; + let best_block_header = match try!(source.get(COL_HEADERS, &best_block_hash)) { + // no best block, nothing to do + None => { + trace!(target: "migration", "No best block header, skipping"); + return Ok(()) + }, + Some(x) => x, + }; + let state_root = HeaderView::new(&best_block_header).state_root(); + + let db = ::std::sync::Arc::new(source); + let mut batch = DBTransaction::new(&db); + + if check_bloom_exists(&*db) { + match check_space_match(&*db) { + Ok(_) => { + // bloom already exists and desired and stored spaces match, nothing to do + trace!(target: "migration", "Bloom already present of the right space, skipping"); + return Ok(()) + }, + Err(wrong_size) => { + // nullify existing bloom entries + trace!(target: "migration", "Clearing old bloom of space {}", &wrong_size); + let mut key = [0u8; 8]; + for idx in 0..(wrong_size as u64/8) { + LittleEndian::write_u64(&mut key, idx); + batch.put(COL_ACCOUNT_BLOOM, &key, &[0u8; 8]); + + if idx % 10000 == 1 { progress.tick(); }; + } + + LittleEndian::write_u64(&mut key, ACCOUNT_BLOOM_SPACE as u64); + batch.put(COL_ACCOUNT_BLOOM, ACCOUNT_BLOOM_SPACE_KEY, &key); + }, + } + } + + println!("Adding/expanding accounts bloom (one-time upgrade)"); + let bloom_journal = { + let mut bloom = Bloom::new(ACCOUNT_BLOOM_SPACE, DEFAULT_ACCOUNT_PRESET); + // no difference what algorithm is passed, since there will be no writes + let state_db = journaldb::new( + db.clone(), + journaldb::Algorithm::OverlayRecent, + COL_STATE); + let account_trie = try!(TrieDB::new(state_db.as_hashdb(), &state_root).map_err(|e| Error::Custom(format!("Cannot open trie: {:?}", e)))); + for item in try!(account_trie.iter().map_err(|_| Error::MigrationImpossible)) { + let (ref account_key, _) = try!(item.map_err(|_| Error::MigrationImpossible)); + let account_key_hash = H256::from_slice(&account_key); + bloom.set(&*account_key_hash); + } + + bloom.drain_journal() + }; + + trace!(target: "migration", "Generated {} bloom updates", bloom_journal.entries.len()); + + try!(StateDB::commit_bloom(&mut batch, bloom_journal).map_err(|_| Error::Custom("Failed to commit bloom".to_owned()))); + try!(db.write(batch)); + + trace!(target: "migration", "Finished bloom update"); + + Ok(()) +} diff --git a/ethcore/src/state_db.rs b/ethcore/src/state_db.rs index 8724a5d73e9..6cdf01098aa 100644 --- a/ethcore/src/state_db.rs +++ b/ethcore/src/state_db.rs @@ -26,10 +26,11 @@ use bloom_journal::{Bloom, BloomJournal}; use db::COL_ACCOUNT_BLOOM; use byteorder::{LittleEndian, ByteOrder}; -pub const ACCOUNT_BLOOM_SPACE: usize = 1048576; -pub const DEFAULT_ACCOUNT_PRESET: usize = 1000000; +pub const ACCOUNT_BLOOM_SPACE: usize = 4 * 1048576; +pub const DEFAULT_ACCOUNT_PRESET: usize = 100 * 1000000; pub const ACCOUNT_BLOOM_HASHCOUNT_KEY: &'static [u8] = b"account_hash_count"; +pub const ACCOUNT_BLOOM_SPACE_KEY: &'static [u8] = b"account_space"; const STATE_CACHE_BLOCKS: usize = 8; diff --git a/parity/migration.rs b/parity/migration.rs index 26bb606bcfb..a75e576a648 100644 --- a/parity/migration.rs +++ b/parity/migration.rs @@ -220,6 +220,12 @@ fn exists(path: &Path) -> bool { fs::metadata(path).is_ok() } +// in-place upgrades that do nothing when called repeatedly +fn run_inplace_upgrades(path: &Path) -> Result<(), Error> { + try!(migrations::upgrade_account_bloom(path)); + Ok(()) +} + /// Migrates the database. pub fn migrate(path: &Path, pruning: Algorithm, compaction_profile: CompactionProfile) -> Result<(), Error> { // read version file. @@ -233,6 +239,9 @@ pub fn migrate(path: &Path, pruning: Algorithm, compaction_profile: CompactionPr // We are in the latest version, yay! if version == CURRENT_VERSION { + // run any inplace migrations for the up-to-date database + try!(run_inplace_upgrades(consolidated_database_path(path).as_path())); + return Ok(()) } @@ -265,7 +274,12 @@ pub fn migrate(path: &Path, pruning: Algorithm, compaction_profile: CompactionPr } // update version file. - update_version(path) + try!(update_version(path)); + + // run any inplace migrations for the fully upgraded database + try!(run_inplace_upgrades(consolidated_database_path(path).as_path())); + + Ok(()) } /// Old migrations utilities