diff --git a/Cargo.lock b/Cargo.lock index 246f9bfd376..743e8f8c7b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -313,23 +313,24 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.49.2" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "846a1fba6535362a01487ef6b10f0275faa12e5c5d835c5c1c627aabc46ccbd6" +checksum = "6bb26d6a69a335b8cb0e7c7e9775cd5666611dc50a37177c3f2cedcfc040e8c8" dependencies = [ "bitflags", "cexpr", "cfg-if", "clang-sys", "clap", - "env_logger 0.6.2", - "fxhash", + "env_logger 0.7.1", "lazy_static", + "lazycell", "log", "peeking_take_while", - "proc-macro2 0.4.20", - "quote 0.6.8", + "proc-macro2 1.0.8", + "quote 1.0.2", "regex", + "rustc-hash", "shlex", "which", ] @@ -534,11 +535,11 @@ dependencies = [ [[package]] name = "cexpr" -version = "0.3.6" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" dependencies = [ - "nom", + "nom 5.1.1", ] [[package]] @@ -568,9 +569,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "0.28.1" +version = "0.29.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +checksum = "f92986241798376849e1a007827041fed9bb36195822c2049d18e174420e0534" dependencies = [ "glob", "libc", @@ -1089,6 +1090,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "error-chain" version = "0.12.0" @@ -1217,7 +1231,7 @@ dependencies = [ "keccak-hash", "kvdb", "kvdb-memorydb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "lazy_static", "log", "machine", @@ -1583,7 +1597,7 @@ dependencies = [ "ethcore-sync", "ethereum-types", "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "log", "snapshot", "spec", @@ -2166,9 +2180,9 @@ checksum = "e8734b0cfd3bc3e101ec59100e101c2eecd19282202e87808b3037b442777a83" [[package]] name = "humantime" -version = "1.1.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" dependencies = [ "quick-error", ] @@ -2599,6 +2613,25 @@ dependencies = [ "smallvec 1.2.0", ] +[[package]] +name = "kvdb-rocksdb" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fecd50b14a534125228d7039951f92aaff742aff151c04546347aba4d3b4fbc" +dependencies = [ + "fs-swap", + "interleaved-ordered", + "kvdb", + "log", + "num_cpus", + "owning_ref 0.4.0", + "parity-util-mem", + "parking_lot 0.10.0", + "regex", + "rocksdb", + "smallvec 1.2.0", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2636,9 +2669,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "6.2.4" +version = "6.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a0785e816e1e11e7599388a492c61ef80ddc2afc91e313e61662cce537809be" +checksum = "4e3b727e2dd20ec2fb7ed93f23d9fd5328a0871185485ebdaff007b47d3e27e4" dependencies = [ "bindgen", "cc", @@ -2855,7 +2888,7 @@ name = "migration-rocksdb" version = "0.1.0" dependencies = [ "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "log", "maplit", "tempdir", @@ -3019,6 +3052,16 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b471253da97532da4b61552249c521e01e736071f71c1a4f7ebbfbf0a06aad6" +dependencies = [ + "memchr", + "version_check 0.9.1", +] + [[package]] name = "null-engine" version = "0.1.0" @@ -3169,7 +3212,7 @@ dependencies = [ "jsonrpc-core", "keccak-hash", "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "log", "migration-rocksdb", "node-filter", @@ -3468,7 +3511,7 @@ dependencies = [ "jsonrpc-server-utils", "keccak-hash", "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.5.0", "lazy_static", "libsecp256k1", "log", @@ -4321,6 +4364,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hex" version = "1.0.0" @@ -4631,7 +4680,7 @@ dependencies = [ "keccak-hash", "keccak-hasher 0.1.1", "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "lazy_static", "log", "num_cpus", @@ -4677,7 +4726,7 @@ dependencies = [ "keccak-hash", "keccak-hasher 0.1.1", "kvdb", - "kvdb-rocksdb", + "kvdb-rocksdb 0.6.0", "lazy_static", "log", "parity-bytes", @@ -5783,16 +5832,15 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bb43f70885151e629e2a19ce9e50bd730fd436cfd4b666894c9ce4de9141164" dependencies = [ - "nom", + "nom 4.2.3", ] [[package]] name = "which" -version = "2.0.1" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" dependencies = [ - "failure", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index c6398e07991..37388ad3e5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ journaldb = { path = "util/journaldb" } jsonrpc-core = "14.0.3" keccak-hash = "0.4.0" kvdb = "0.4.0" -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" log = "0.4" migration-rocksdb = { path = "util/migration-rocksdb" } node-filter = { path = "ethcore/node-filter" } diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index c6d09a6ba9c..d11a6df2d69 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -33,7 +33,7 @@ journaldb = { path = "../util/journaldb" } keccak-hash = "0.4.0" kvdb = "0.4.0" kvdb-memorydb = { version = "0.4.0", optional = true } -kvdb-rocksdb = { version = "0.5.0", optional = true } +kvdb-rocksdb = { version = "0.6.0", optional = true } lazy_static = { version = "1.3", optional = true } log = "0.4" machine = { path = "./machine" } @@ -78,7 +78,7 @@ ethjson = { path = "../json", features = ["test-helpers"] } parity-crypto = { version = "0.5.0", features = ["publickey"] } fetch = { path = "../util/fetch" } kvdb-memorydb = "0.4.0" -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" lazy_static = "1.3" machine = { path = "./machine", features = ["test-helpers"] } parity-runtime = "0.1.1" diff --git a/ethcore/service/Cargo.toml b/ethcore/service/Cargo.toml index d9b158682fc..5e8c0c789b2 100644 --- a/ethcore/service/Cargo.toml +++ b/ethcore/service/Cargo.toml @@ -23,5 +23,5 @@ trace-time = "0.1" [dev-dependencies] ethcore = { path = "..", features = ["test-helpers"] } ethcore-db = { path = "../db" } -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" tempdir = "0.3" diff --git a/ethcore/snapshot/Cargo.toml b/ethcore/snapshot/Cargo.toml index 6e132b5259c..c9ac9e045d3 100644 --- a/ethcore/snapshot/Cargo.toml +++ b/ethcore/snapshot/Cargo.toml @@ -53,7 +53,7 @@ ethabi-contract = "9.0.0" ethabi-derive = "9.0.1" ethcore = { path = "..", features = ["test-helpers"] } ethkey = { path = "../../accounts/ethkey" } -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" lazy_static = { version = "1.3" } spec = { path = "../spec" } tempdir = "0.3" diff --git a/ethcore/snapshot/snapshot-tests/Cargo.toml b/ethcore/snapshot/snapshot-tests/Cargo.toml index 2761a815892..56be57d4379 100644 --- a/ethcore/snapshot/snapshot-tests/Cargo.toml +++ b/ethcore/snapshot/snapshot-tests/Cargo.toml @@ -24,7 +24,7 @@ journaldb = { path = "../../../util/journaldb" } keccak-hash = "0.4.0" keccak-hasher = { path = "../../../util/keccak-hasher" } kvdb = "0.4.0" -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" log = "0.4.8" parking_lot = "0.10.0" parity-crypto = { version = "0.5.0", features = ["publickey"] } diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 4e6c24c45dc..916c370bc04 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -109,7 +109,7 @@ use types::{ BlockNumber, call_analytics::CallAnalytics, chain_notify::{ChainMessageType, ChainRoute, NewBlocks}, - client_types::{ClientReport, Mode, StateResult}, + client_types::{ClientReport, IoStats, Mode, StateResult}, encoded, engines::{ epoch::{PendingTransition, Transition as EpochTransition}, @@ -1096,7 +1096,19 @@ impl Client { /// Get the report. pub fn report(&self) -> ClientReport { let mut report = self.report.read().clone(); - report.state_db_mem = self.state_db.read().mem_used(); + let state_db = self.state_db.read(); + report.state_db_mem = state_db.mem_used(); + let io_stats = state_db.journal_db().io_stats(); + report.io_stats = IoStats { + transactions: io_stats.transactions, + reads: io_stats.reads, + cache_reads: io_stats.cache_reads, + writes: io_stats.writes, + bytes_read: io_stats.bytes_read, + cache_read_bytes: io_stats.cache_read_bytes, + bytes_written: io_stats.bytes_written, + }; + report } diff --git a/ethcore/types/src/client_types.rs b/ethcore/types/src/client_types.rs index 0d5f68dc8f9..9a163f4bc5a 100644 --- a/ethcore/types/src/client_types.rs +++ b/ethcore/types/src/client_types.rs @@ -62,6 +62,27 @@ pub struct ClientReport { pub gas_processed: U256, /// Memory used by state DB pub state_db_mem: usize, + /// I/O statistics for the state DB. + pub io_stats: IoStats, +} + +/// I/O statistics. +#[derive(Default, Debug, Clone, Eq, PartialEq)] +pub struct IoStats { + /// Number of transaction. + pub transactions: u64, + /// Number of read operations. + pub reads: u64, + /// Number of reads resulted in a read from cache. + pub cache_reads: u64, + /// Number of write operations. + pub writes: u64, + /// Number of bytes read. + pub bytes_read: u64, + /// Number of bytes read from cache. + pub cache_read_bytes: u64, + /// Number of bytes write. + pub bytes_written: u64, } impl ClientReport { diff --git a/ethcore/verification/src/queue/mod.rs b/ethcore/verification/src/queue/mod.rs index 341682172cc..194311c1fae 100644 --- a/ethcore/verification/src/queue/mod.rs +++ b/ethcore/verification/src/queue/mod.rs @@ -234,7 +234,7 @@ impl VerificationQueue { let number_of_threads = if scale_verifiers { max_verifiers } else { - cmp::min(default_amount, max_verifiers) + default_amount }; let state = Arc::new((Mutex::new(State::Work(default_amount)), Condvar::new())); diff --git a/parity/informant.rs b/parity/informant.rs index 8710803093f..d4e81deda2d 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -257,6 +257,14 @@ impl Informant { (diffed, full_report) }; + debug!( + target: "io_stats", + "{} reads, {} writes, {} transactions", + client_report.io_stats.reads, + client_report.io_stats.writes, + client_report.io_stats.transactions, + ); + let Report { importing, chain_info, diff --git a/util/journaldb/src/archivedb.rs b/util/journaldb/src/archivedb.rs index 0da2dac0c49..1b68e32f6fc 100644 --- a/util/journaldb/src/archivedb.rs +++ b/util/journaldb/src/archivedb.rs @@ -102,6 +102,10 @@ impl JournalDB for ArchiveDB { Box::new(self.clone()) } + fn io_stats(&self) -> kvdb::IoStats { + self.backing.io_stats(kvdb::IoStatsKind::SincePrevious) + } + fn mem_used(&self) -> usize { self.overlay.malloc_size_of() } diff --git a/util/journaldb/src/earlymergedb.rs b/util/journaldb/src/earlymergedb.rs index 9fc873db1c2..0f2a6130657 100644 --- a/util/journaldb/src/earlymergedb.rs +++ b/util/journaldb/src/earlymergedb.rs @@ -322,6 +322,10 @@ impl JournalDB for EarlyMergeDB { Box::new(self.clone()) } + fn io_stats(&self) -> kvdb::IoStats { + self.backing.io_stats(kvdb::IoStatsKind::SincePrevious) + } + fn is_empty(&self) -> bool { self.backing.get(self.column, &LATEST_ERA_KEY).expect("Low level database error").is_none() } diff --git a/util/journaldb/src/lib.rs b/util/journaldb/src/lib.rs index b2dea0b205c..e18e94646f6 100644 --- a/util/journaldb/src/lib.rs +++ b/util/journaldb/src/lib.rs @@ -45,6 +45,9 @@ pub trait JournalDB: HashDB { /// Returns heap memory size used fn mem_used(&self) -> usize; + /// Returns the I/O statistics. + fn io_stats(&self) -> kvdb::IoStats; + /// Returns the size of journalled state in memory. /// This function has a considerable speed requirement -- /// it must be fast enough to call several times per block imported. diff --git a/util/journaldb/src/overlayrecentdb.rs b/util/journaldb/src/overlayrecentdb.rs index 11643fdac2b..62e7d43e8e0 100644 --- a/util/journaldb/src/overlayrecentdb.rs +++ b/util/journaldb/src/overlayrecentdb.rs @@ -244,6 +244,10 @@ impl JournalDB for OverlayRecentDB { Box::new(self.clone()) } + fn io_stats(&self) -> kvdb::IoStats { + self.backing.io_stats(kvdb::IoStatsKind::SincePrevious) + } + fn mem_used(&self) -> usize { let mut ops = new_malloc_size_ops(); let mut mem = self.transaction_overlay.size_of(&mut ops); diff --git a/util/journaldb/src/refcounteddb.rs b/util/journaldb/src/refcounteddb.rs index 891c3c5860d..ea278a6ac18 100644 --- a/util/journaldb/src/refcounteddb.rs +++ b/util/journaldb/src/refcounteddb.rs @@ -98,6 +98,10 @@ impl JournalDB for RefCountedDB { Box::new(self.clone()) } + fn io_stats(&self) -> kvdb::IoStats { + self.backing.io_stats(kvdb::IoStatsKind::SincePrevious) + } + fn mem_used(&self) -> usize { let mut ops = new_malloc_size_ops(); self.inserts.size_of(&mut ops) + self.removes.size_of(&mut ops) diff --git a/util/migration-rocksdb/Cargo.toml b/util/migration-rocksdb/Cargo.toml index 1748b4b1586..5d5cd40d8ca 100644 --- a/util/migration-rocksdb/Cargo.toml +++ b/util/migration-rocksdb/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] log = "0.4" kvdb = "0.4.0" -kvdb-rocksdb = "0.5.0" +kvdb-rocksdb = "0.6.0" [dev-dependencies] tempdir = "0.3"