From f231828ebe103f518c5af773f4260d5670e848a9 Mon Sep 17 00:00:00 2001 From: 5chdn <5chdn@users.noreply.github.com> Date: Tue, 27 Nov 2018 13:42:06 +0100 Subject: [PATCH 01/18] version: bump beta to 2.2.2 --- Cargo.lock | 12 ++++++------ Cargo.toml | 2 +- util/version/Cargo.toml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eaa74650e59..f50cfd38e48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2152,7 +2152,7 @@ version = "1.12.0" dependencies = [ "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", - "parity-ethereum 2.2.1", + "parity-ethereum 2.2.2", ] [[package]] @@ -2168,7 +2168,7 @@ dependencies = [ [[package]] name = "parity-ethereum" -version = "2.2.1" +version = "2.2.2" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2217,7 +2217,7 @@ dependencies = [ "parity-rpc-client 1.4.0", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.1", + "parity-version 2.2.2", "parity-whisper 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2372,7 +2372,7 @@ dependencies = [ "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.1", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2469,7 +2469,7 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.12.0", "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-version 2.2.1", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2479,7 +2479,7 @@ dependencies = [ [[package]] name = "parity-version" -version = "2.2.1" +version = "2.2.2" dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index e0f197ecc50..3af2dbab831 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ description = "Parity Ethereum client" name = "parity-ethereum" # NOTE Make sure to update util/version/Cargo.toml as well -version = "2.2.1" +version = "2.2.2" license = "GPL-3.0" authors = ["Parity Technologies "] diff --git a/util/version/Cargo.toml b/util/version/Cargo.toml index 854d04c852e..f46a0d54d83 100644 --- a/util/version/Cargo.toml +++ b/util/version/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "parity-version" # NOTE: this value is used for Parity Ethereum version string (via env CARGO_PKG_VERSION) -version = "2.2.1" +version = "2.2.2" authors = ["Parity Technologies "] build = "build.rs" From 6a40a463be695bfb0858dd45ff28feedb14e8808 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Fri, 16 Nov 2018 13:00:34 +0000 Subject: [PATCH 02/18] Add experimental RPCs flag (#9928) * WiP * Enable experimental RPCs. --- parity/cli/mod.rs | 35 +++++---- parity/configuration.rs | 3 + parity/rpc_apis.rs | 6 +- parity/run.rs | 3 + rpc/src/v1/helpers/errors.rs | 13 +++ rpc/src/v1/impls/personal.rs | 13 ++- rpc/src/v1/tests/mocked/personal.rs | 118 +++++++++++++++++++++++++++- 7 files changed, 173 insertions(+), 18 deletions(-) diff --git a/parity/cli/mod.rs b/parity/cli/mod.rs index 5c3bddf6c36..17e261800d5 100644 --- a/parity/cli/mod.rs +++ b/parity/cli/mod.rs @@ -467,6 +467,10 @@ usage! { "--no-jsonrpc", "Disable the HTTP JSON-RPC API server.", + FLAG flag_jsonrpc_experimental: (bool) = false, or |c: &Config| c.rpc.as_ref()?.experimental_rpcs.clone(), + "--jsonrpc-experimental", + "Enable experimental RPCs. Enable to have access to methods from unfinalised EIPs in all namespaces", + ARG arg_jsonrpc_port: (u16) = 8545u16, or |c: &Config| c.rpc.as_ref()?.port.clone(), "--jsonrpc-port=[PORT]", "Specify the port portion of the HTTP JSON-RPC API server.", @@ -1141,7 +1145,7 @@ struct Operating { no_persistent_txqueue: Option, no_hardcoded_sync: Option, - #[serde(rename="public_node")] + #[serde(rename = "public_node")] _legacy_public_node: Option, } @@ -1173,15 +1177,15 @@ struct PrivateTransactions { struct Ui { path: Option, - #[serde(rename="force")] + #[serde(rename = "force")] _legacy_force: Option, - #[serde(rename="disable")] + #[serde(rename = "disable")] _legacy_disable: Option, - #[serde(rename="port")] + #[serde(rename = "port")] _legacy_port: Option, - #[serde(rename="interface")] + #[serde(rename = "interface")] _legacy_interface: Option, - #[serde(rename="hosts")] + #[serde(rename = "hosts")] _legacy_hosts: Option>, } @@ -1219,6 +1223,7 @@ struct Rpc { server_threads: Option, processing_threads: Option, max_payload: Option, + experimental_rpcs: Option, } #[derive(Default, Debug, PartialEq, Deserialize)] @@ -1244,21 +1249,21 @@ struct Ipc { #[derive(Default, Debug, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] struct Dapps { - #[serde(rename="disable")] + #[serde(rename = "disable")] _legacy_disable: Option, - #[serde(rename="port")] + #[serde(rename = "port")] _legacy_port: Option, - #[serde(rename="interface")] + #[serde(rename = "interface")] _legacy_interface: Option, - #[serde(rename="hosts")] + #[serde(rename = "hosts")] _legacy_hosts: Option>, - #[serde(rename="cors")] + #[serde(rename = "cors")] _legacy_cors: Option, - #[serde(rename="path")] + #[serde(rename = "path")] _legacy_path: Option, - #[serde(rename="user")] + #[serde(rename = "user")] _legacy_user: Option, - #[serde(rename="pass")] + #[serde(rename = "pass")] _legacy_pass: Option, } @@ -1676,6 +1681,7 @@ mod tests { // -- API and Console Options // RPC flag_no_jsonrpc: false, + flag_jsonrpc_experimental: false, arg_jsonrpc_port: 8545u16, arg_jsonrpc_interface: "local".into(), arg_jsonrpc_cors: "null".into(), @@ -1958,6 +1964,7 @@ mod tests { server_threads: None, processing_threads: None, max_payload: None, + experimental_rpcs: None, }), ipc: Some(Ipc { disable: None, diff --git a/parity/configuration.rs b/parity/configuration.rs index 8ec2fa88c24..661a8d9b2df 100644 --- a/parity/configuration.rs +++ b/parity/configuration.rs @@ -138,6 +138,7 @@ impl Configuration { let compaction = self.args.arg_db_compaction.parse()?; let warp_sync = !self.args.flag_no_warp; let geth_compatibility = self.args.flag_geth; + let experimental_rpcs = self.args.flag_jsonrpc_experimental; let ipfs_conf = self.ipfs_config(); let secretstore_conf = self.secretstore_config()?; let format = self.format()?; @@ -377,6 +378,7 @@ impl Configuration { warp_sync: warp_sync, warp_barrier: self.args.arg_warp_barrier, geth_compatibility: geth_compatibility, + experimental_rpcs, net_settings: self.network_settings()?, ipfs_conf: ipfs_conf, secretstore_conf: secretstore_conf, @@ -1418,6 +1420,7 @@ mod tests { compaction: Default::default(), vm_type: Default::default(), geth_compatibility: false, + experimental_rpcs: false, net_settings: Default::default(), ipfs_conf: Default::default(), secretstore_conf: Default::default(), diff --git a/parity/rpc_apis.rs b/parity/rpc_apis.rs index d07b25a8d28..9839b70d27f 100644 --- a/parity/rpc_apis.rs +++ b/parity/rpc_apis.rs @@ -228,6 +228,7 @@ pub struct FullDependencies { pub net_service: Arc, pub updater: Arc, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub ws_address: Option, pub fetch: FetchClient, pub executor: Executor, @@ -317,7 +318,7 @@ impl FullDependencies { } }, Api::Personal => { - handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility).to_delegate()); + handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility, self.experimental_rpcs).to_delegate()); }, Api::Signer => { handler.extend_with(SignerClient::new(&self.secret_store, dispatcher.clone(), &self.signer_service, self.executor.clone()).to_delegate()); @@ -438,6 +439,7 @@ pub struct LightDependencies { pub ws_address: Option, pub fetch: FetchClient, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub executor: Executor, pub whisper_rpc: Option<::whisper::RpcFactory>, pub private_tx_service: Option>, @@ -531,7 +533,7 @@ impl LightDependencies { handler.extend_with(EthPubSub::to_delegate(client)); }, Api::Personal => { - handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility).to_delegate()); + handler.extend_with(PersonalClient::new(&self.secret_store, dispatcher.clone(), self.geth_compatibility, self.experimental_rpcs).to_delegate()); }, Api::Signer => { handler.extend_with(SignerClient::new(&self.secret_store, dispatcher.clone(), &self.signer_service, self.executor.clone()).to_delegate()); diff --git a/parity/run.rs b/parity/run.rs index 3a176057a77..03dbcaffb0b 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -115,6 +115,7 @@ pub struct RunCmd { pub compaction: DatabaseCompactionProfile, pub vm_type: VMType, pub geth_compatibility: bool, + pub experimental_rpcs: bool, pub net_settings: NetworkSettings, pub ipfs_conf: ipfs::Configuration, pub secretstore_conf: secretstore::Configuration, @@ -312,6 +313,7 @@ fn execute_light_impl(cmd: RunCmd, logger: Arc) -> Result(cmd: RunCmd, logger: Arc, on_client_rq: net_service: manage_network.clone(), updater: updater.clone(), geth_compatibility: cmd.geth_compatibility, + experimental_rpcs: cmd.experimental_rpcs, ws_address: cmd.ws_conf.address(), fetch: fetch.clone(), executor: runtime.executor(), diff --git a/rpc/src/v1/helpers/errors.rs b/rpc/src/v1/helpers/errors.rs index f177e251568..38279d3b72f 100644 --- a/rpc/src/v1/helpers/errors.rs +++ b/rpc/src/v1/helpers/errors.rs @@ -53,6 +53,7 @@ mod codes { pub const FETCH_ERROR: i64 = -32060; pub const NO_LIGHT_PEERS: i64 = -32065; pub const DEPRECATED: i64 = -32070; + pub const EXPERIMENTAL_RPC: i64 = -32071; } pub fn unimplemented(details: Option) -> Error { @@ -500,3 +501,15 @@ pub fn on_demand_others(err: &OnDemandError) -> Error { } } +/// Returns a descriptive error in case experimental RPCs are not enabled. +pub fn require_experimental(allow_experimental_rpcs: bool, eip: &str) -> Result<(), Error> { + if allow_experimental_rpcs { + Ok(()) + } else { + Err(Error { + code: ErrorCode::ServerError(codes::EXPERIMENTAL_RPC), + message: format!("This method is not part of the official RPC API yet (EIP-{}). Run with `--jsonrpc-experimental` to enable it.", eip), + data: Some(Value::String(format!("See EIP: https://eips.ethereum.org/EIPS/eip-{}", eip))), + }) + } +} diff --git a/rpc/src/v1/impls/personal.rs b/rpc/src/v1/impls/personal.rs index 4da0c1023e9..e42a91b1f26 100644 --- a/rpc/src/v1/impls/personal.rs +++ b/rpc/src/v1/impls/personal.rs @@ -47,15 +47,22 @@ pub struct PersonalClient { accounts: Arc, dispatcher: D, allow_perm_unlock: bool, + allow_experimental_rpcs: bool, } impl PersonalClient { /// Creates new PersonalClient - pub fn new(accounts: &Arc, dispatcher: D, allow_perm_unlock: bool) -> Self { + pub fn new( + accounts: &Arc, + dispatcher: D, + allow_perm_unlock: bool, + allow_experimental_rpcs: bool, + ) -> Self { PersonalClient { accounts: accounts.clone(), dispatcher, allow_perm_unlock, + allow_experimental_rpcs, } } } @@ -154,6 +161,8 @@ impl Personal for PersonalClient { } fn sign_191(&self, version: EIP191Version, data: Value, account: RpcH160, password: String) -> BoxFuture { + try_bf!(errors::require_experimental(self.allow_experimental_rpcs, "191")); + let data = try_bf!(eip191::hash_message(version, data)); let dispatcher = self.dispatcher.clone(); let accounts = self.accounts.clone(); @@ -174,6 +183,8 @@ impl Personal for PersonalClient { } fn sign_typed_data(&self, typed_data: EIP712, account: RpcH160, password: String) -> BoxFuture { + try_bf!(errors::require_experimental(self.allow_experimental_rpcs, "712")); + let data = match hash_structured_data(typed_data) { Ok(d) => d, Err(err) => return Box::new(future::err(errors::invalid_call_data(err.kind()))), diff --git a/rpc/src/v1/tests/mocked/personal.rs b/rpc/src/v1/tests/mocked/personal.rs index 5098d807d94..143b19fa8d6 100644 --- a/rpc/src/v1/tests/mocked/personal.rs +++ b/rpc/src/v1/tests/mocked/personal.rs @@ -57,6 +57,16 @@ fn miner_service() -> Arc { } fn setup() -> PersonalTester { + setup_with(Config { + allow_experimental_rpcs: true + }) +} + +struct Config { + pub allow_experimental_rpcs: bool, +} + +fn setup_with(c: Config) -> PersonalTester { let runtime = Runtime::with_thread_count(1); let accounts = accounts_provider(); let client = blockchain_client(); @@ -64,7 +74,7 @@ fn setup() -> PersonalTester { let reservations = Arc::new(Mutex::new(nonce::Reservations::new(runtime.executor()))); let dispatcher = FullDispatcher::new(client, miner.clone(), reservations, 50); - let personal = PersonalClient::new(&accounts, dispatcher, false); + let personal = PersonalClient::new(&accounts, dispatcher, false, c.allow_experimental_rpcs); let mut io = IoHandler::default(); io.extend_with(personal.to_delegate()); @@ -418,3 +428,109 @@ fn sign_eip191_structured_data() { let response = tester.io.handle_request_sync(&request).unwrap(); assert_eq!(response, expected) } + +#[test] +fn sign_structured_data() { + let tester = setup(); + let secret: Secret = keccak("cow").into(); + let address = tester.accounts.insert_account(secret, &"lol".into()).unwrap(); + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_signTypedData", + "params": [ + { + "primaryType": "Mail", + "domain": { + "name": "Ether Mail", + "version": "1", + "chainId": "0x1", + "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC" + }, + "message": { + "from": { + "name": "Cow", + "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826" + }, + "to": { + "name": "Bob", + "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB" + }, + "contents": "Hello, Bob!" + }, + "types": { + "EIP712Domain": [ + { "name": "name", "type": "string" }, + { "name": "version", "type": "string" }, + { "name": "chainId", "type": "uint256" }, + { "name": "verifyingContract", "type": "address" } + ], + "Person": [ + { "name": "name", "type": "string" }, + { "name": "wallet", "type": "address" } + ], + "Mail": [ + { "name": "from", "type": "Person" }, + { "name": "to", "type": "Person" }, + { "name": "contents", "type": "string" } + ] + } + }, + ""#.to_owned() + &format!("0x{:x}", address) + r#"", + "lol" + ], + "id": 1 + }"#; + let expected = r#"{"jsonrpc":"2.0","result":"0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c","id":1}"#; + let response = tester.io.handle_request_sync(&request).unwrap(); + assert_eq!(response, expected) +} + +#[test] +fn should_disable_experimental_apis() { + // given + let tester = setup_with(Config { + allow_experimental_rpcs: false, + }); + + // when + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_sign191", + "params": [ + "0x01", + {}, + "0x1234567891234567891234567891234567891234", + "lol" + ], + "id": 1 + }"#; + let r1 = tester.io.handle_request_sync(&request).unwrap(); + let request = r#"{ + "jsonrpc": "2.0", + "method": "personal_signTypedData", + "params": [ + { + "types": {}, + "message": {}, + "domain": { + "name": "", + "version": "1", + "chainId": "0x1", + "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC" + }, + "primaryType": "" + }, + "0x1234567891234567891234567891234678912344", + "lol" + ], + "id": 1 + }"#; + let r2 = tester.io.handle_request_sync(&request).unwrap(); + + // then + let expected = r#"{"jsonrpc":"2.0","error":{"code":-32071,"message":"This method is not part of the official RPC API yet (EIP-191). Run with `--jsonrpc-experimental` to enable it.","data":"See EIP: https://eips.ethereum.org/EIPS/eip-191"},"id":1}"#; + assert_eq!(r1, expected); + + let expected = r#"{"jsonrpc":"2.0","error":{"code":-32071,"message":"This method is not part of the official RPC API yet (EIP-712). Run with `--jsonrpc-experimental` to enable it.","data":"See EIP: https://eips.ethereum.org/EIPS/eip-712"},"id":1}"#; + assert_eq!(r2, expected); +} From 3a3e321dc5c4ea73271bdea6d615a172e6380822 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Sun, 18 Nov 2018 00:06:34 +0100 Subject: [PATCH 03/18] Keep existing blocks when restoring a Snapshot (#8643) * Rename db_restore => client * First step: make it compile! * Second step: working implementation! * Refactoring * Fix tests * PR Grumbles * PR Grumbles WIP * Migrate ancient blocks interating backward * Early return in block migration if snapshot is aborted * Remove RwLock getter (PR Grumble I) * Remove dependency on `Client`: only used Traits * Add test for recovering aborted snapshot recovery * Add test for migrating old blocks * Fix build * PR Grumble I * PR Grumble II * PR Grumble III * PR Grumble IV * PR Grumble V * PR Grumble VI * Fix one test * Fix test * PR Grumble * PR Grumbles * PR Grumbles II * Fix tests * Release RwLock earlier * Revert Cargo.lock * Update _update ancient block_ logic: set local in `commit` * Update typo in ethcore/src/snapshot/service.rs Co-Authored-By: ngotchac --- Cargo.lock | 11 ++ ethcore/Cargo.toml | 1 + ethcore/light/src/provider.rs | 4 +- ethcore/service/src/service.rs | 2 +- ethcore/src/blockchain/blockchain.rs | 99 +++++++++-- ethcore/src/client/client.rs | 24 ++- ethcore/src/client/test_client.rs | 8 +- ethcore/src/client/traits.rs | 8 +- ethcore/src/lib.rs | 3 + ethcore/src/snapshot/error.rs | 3 + ethcore/src/snapshot/mod.rs | 2 +- ethcore/src/snapshot/service.rs | 146 ++++++++++++++--- ethcore/src/snapshot/tests/service.rs | 227 ++++++++++++++++++++++++-- ethcore/src/test_helpers.rs | 60 ++++--- ethcore/sync/src/chain/mod.rs | 6 + ethcore/sync/src/chain/supplier.rs | 3 +- rpc/src/v1/impls/parity.rs | 2 +- 17 files changed, 518 insertions(+), 91 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f50cfd38e48..dfd911b2593 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,6 +460,15 @@ dependencies = [ "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "env_logger" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "env_logger" version = "0.5.13" @@ -558,6 +567,7 @@ dependencies = [ "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "common-types 0.1.0", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi 6.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethabi-contract 6.0.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4083,6 +4093,7 @@ dependencies = [ "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" "checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" +"checksum env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3ddf21e73e016298f5cb37d6ef8e8da8e39f91f9ec8b0df44b7deb16a9f8cd5b" "checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" "checksum error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "07e791d3be96241c77c43846b665ef1384606da2cd2a48730abe606a12906e02" "checksum eth-secp256k1 0.5.7 (git+https://github.com/paritytech/rust-secp256k1)" = "" diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index 31e8da942a0..989c705d1c7 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -76,6 +76,7 @@ hardware-wallet = { path = "../hw" } fake-hardware-wallet = { path = "../util/fake-hardware-wallet" } [dev-dependencies] +env_logger = "0.4" tempdir = "0.3" trie-standardmap = "0.1" diff --git a/ethcore/light/src/provider.rs b/ethcore/light/src/provider.rs index 72344620653..bfd15dadc28 100644 --- a/ethcore/light/src/provider.rs +++ b/ethcore/light/src/provider.rs @@ -176,8 +176,8 @@ impl Provider for T { } fn block_receipts(&self, req: request::CompleteReceiptsRequest) -> Option { - BlockChainClient::encoded_block_receipts(self, &req.hash) - .map(|x| ::request::ReceiptsResponse { receipts: ::rlp::decode_list(&x) }) + BlockChainClient::block_receipts(self, &req.hash) + .map(|x| ::request::ReceiptsResponse { receipts: x.receipts }) } fn account_proof(&self, req: request::CompleteAccountRequest) -> Option { diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index a6bbc4e1073..1763b8fd5ed 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -117,7 +117,7 @@ impl ClientService { pruning: pruning, channel: io_service.channel(), snapshot_root: snapshot_path.into(), - db_restore: client.clone(), + client: client.clone(), }; let snapshot = Arc::new(SnapshotService::new(snapshot_params)?); diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index 5785aa10343..e5af6420861 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -229,6 +229,7 @@ pub struct BlockChain { cache_man: Mutex>, + pending_best_ancient_block: RwLock>>, pending_best_block: RwLock>, pending_block_hashes: RwLock>, pending_block_details: RwLock>, @@ -538,6 +539,7 @@ impl BlockChain { block_receipts: RwLock::new(HashMap::new()), db: db.clone(), cache_man: Mutex::new(cache_man), + pending_best_ancient_block: RwLock::new(None), pending_best_block: RwLock::new(None), pending_block_hashes: RwLock::new(HashMap::new()), pending_block_details: RwLock::new(HashMap::new()), @@ -808,18 +810,7 @@ impl BlockChain { }, is_best); if is_ancient { - let mut best_ancient_block = self.best_ancient_block.write(); - let ancient_number = best_ancient_block.as_ref().map_or(0, |b| b.number); - if self.block_hash(block_number + 1).is_some() { - batch.delete(db::COL_EXTRA, b"ancient"); - *best_ancient_block = None; - } else if block_number > ancient_number { - batch.put(db::COL_EXTRA, b"ancient", &hash); - *best_ancient_block = Some(BestAncientBlock { - hash: hash, - number: block_number, - }); - } + self.set_best_ancient_block(block_number, &hash, batch); } false @@ -860,6 +851,84 @@ impl BlockChain { } } + /// Update the best ancient block to the given hash, after checking that + /// it's directly linked to the currently known best ancient block + pub fn update_best_ancient_block(&self, hash: &H256) { + // Get the block view of the next ancient block (it must + // be in DB at this point) + let block_view = match self.block(hash) { + Some(v) => v, + None => return, + }; + + // So that `best_ancient_block` gets unlocked before calling + // `set_best_ancient_block` + { + // Get the target hash ; if there are no ancient block, + // it means that the chain is already fully linked + // Release the `best_ancient_block` RwLock + let target_hash = { + let best_ancient_block = self.best_ancient_block.read(); + let cur_ancient_block = match *best_ancient_block { + Some(ref b) => b, + None => return, + }; + + // Ensure that the new best ancient block is after the current one + if block_view.number() <= cur_ancient_block.number { + return; + } + + cur_ancient_block.hash.clone() + }; + + let mut block_hash = *hash; + let mut is_linked = false; + + loop { + if block_hash == target_hash { + is_linked = true; + break; + } + + match self.block_details(&block_hash) { + Some(block_details) => { + block_hash = block_details.parent; + }, + None => break, + } + } + + if !is_linked { + trace!(target: "blockchain", "The given block {:x} is not linked to the known ancient block {:x}", hash, target_hash); + return; + } + } + + let mut batch = self.db.key_value().transaction(); + self.set_best_ancient_block(block_view.number(), hash, &mut batch); + self.db.key_value().write(batch).expect("Low level database error."); + } + + /// Set the best ancient block with the given value: private method + /// `best_ancient_block` must not be locked, otherwise a DeadLock would occur + fn set_best_ancient_block(&self, block_number: BlockNumber, block_hash: &H256, batch: &mut DBTransaction) { + let mut pending_best_ancient_block = self.pending_best_ancient_block.write(); + let ancient_number = self.best_ancient_block.read().as_ref().map_or(0, |b| b.number); + if self.block_hash(block_number + 1).is_some() { + trace!(target: "blockchain", "The two ends of the chain have met."); + batch.delete(db::COL_EXTRA, b"ancient"); + *pending_best_ancient_block = Some(None); + } else if block_number > ancient_number { + trace!(target: "blockchain", "Updating the best ancient block to {}.", block_number); + batch.put(db::COL_EXTRA, b"ancient", &block_hash); + *pending_best_ancient_block = Some(Some(BestAncientBlock { + hash: *block_hash, + number: block_number, + })); + } + } + /// Insert an epoch transition. Provide an epoch number being transitioned to /// and epoch transition object. /// @@ -1112,15 +1181,21 @@ impl BlockChain { /// Apply pending insertion updates pub fn commit(&self) { + let mut pending_best_ancient_block = self.pending_best_ancient_block.write(); let mut pending_best_block = self.pending_best_block.write(); let mut pending_write_hashes = self.pending_block_hashes.write(); let mut pending_block_details = self.pending_block_details.write(); let mut pending_write_txs = self.pending_transaction_addresses.write(); + let mut best_ancient_block = self.best_ancient_block.write(); let mut best_block = self.best_block.write(); let mut write_block_details = self.block_details.write(); let mut write_hashes = self.block_hashes.write(); let mut write_txs = self.transaction_addresses.write(); + // update best ancient block + if let Some(block_option) = pending_best_ancient_block.take() { + *best_ancient_block = block_option; + } // update best block if let Some(block) = pending_best_block.take() { *best_block = block; diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index ff09f7c72a5..8412881a22e 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -32,7 +32,7 @@ use kvdb::{DBValue, KeyValueDB, DBTransaction}; // other use ethereum_types::{H256, Address, U256}; use block::{IsBlock, LockedBlock, Drain, ClosedBlock, OpenBlock, enact_verified, SealedBlock}; -use blockchain::{BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert}; +use blockchain::{BlockReceipts, BlockChain, BlockChainDB, BlockProvider, TreeRoute, ImportRoute, TransactionAddress, ExtrasInsert}; use client::ancient_import::AncientVerifier; use client::{ Nonce, Balance, ChainInfo, BlockInfo, CallContract, TransactionInfo, @@ -66,7 +66,7 @@ use ethcore_miner::pool::VerifiedTransaction; use parking_lot::{Mutex, RwLock}; use rand::OsRng; use receipt::{Receipt, LocalizedReceipt}; -use snapshot::{self, io as snapshot_io}; +use snapshot::{self, io as snapshot_io, SnapshotClient}; use spec::Spec; use state_db::StateDB; use state::{self, State}; @@ -1005,6 +1005,16 @@ impl Client { self.importer.miner.clone() } + #[cfg(test)] + pub fn state_db(&self) -> ::parking_lot::RwLockReadGuard { + self.state_db.read() + } + + #[cfg(test)] + pub fn chain(&self) -> Arc { + self.chain.read().clone() + } + /// Replace io channel. Useful for testing. pub fn set_io_channel(&self, io_channel: IoChannel) { *self.io_channel.write() = io_channel; @@ -1817,7 +1827,7 @@ impl BlockChainClient for Client { Some(receipt) } - fn block_receipts(&self, id: BlockId) -> Option> { + fn localized_block_receipts(&self, id: BlockId) -> Option> { let hash = self.block_hash(id)?; let chain = self.chain.read(); @@ -1860,8 +1870,8 @@ impl BlockChainClient for Client { self.state_db.read().journal_db().state(hash) } - fn encoded_block_receipts(&self, hash: &H256) -> Option { - self.chain.read().block_receipts(hash).map(|receipts| ::rlp::encode(&receipts)) + fn block_receipts(&self, hash: &H256) -> Option { + self.chain.read().block_receipts(hash) } fn queue_info(&self) -> BlockQueueInfo { @@ -2406,6 +2416,8 @@ impl ProvingBlockChainClient for Client { } } +impl SnapshotClient for Client {} + impl Drop for Client { fn drop(&mut self) { self.engine.stop(); @@ -2504,7 +2516,7 @@ mod tests { use test_helpers::{generate_dummy_client_with_data}; let client = generate_dummy_client_with_data(2, 2, &[1.into(), 1.into()]); - let receipts = client.block_receipts(BlockId::Latest).unwrap(); + let receipts = client.localized_block_receipts(BlockId::Latest).unwrap(); assert_eq!(receipts.len(), 2); assert_eq!(receipts[0].transaction_index, 0); diff --git a/ethcore/src/client/test_client.rs b/ethcore/src/client/test_client.rs index c3d8c127b08..3c422c49a58 100644 --- a/ethcore/src/client/test_client.rs +++ b/ethcore/src/client/test_client.rs @@ -686,7 +686,7 @@ impl BlockChainClient for TestBlockChainClient { self.receipts.read().get(&id).cloned() } - fn block_receipts(&self, _id: BlockId) -> Option> { + fn localized_block_receipts(&self, _id: BlockId) -> Option> { Some(self.receipts.read().values().cloned().collect()) } @@ -789,16 +789,14 @@ impl BlockChainClient for TestBlockChainClient { None } - fn encoded_block_receipts(&self, hash: &H256) -> Option { + fn block_receipts(&self, hash: &H256) -> Option { // starts with 'f' ? if *hash > H256::from("f000000000000000000000000000000000000000000000000000000000000000") { let receipt = BlockReceipts::new(vec![Receipt::new( TransactionOutcome::StateRoot(H256::zero()), U256::zero(), vec![])]); - let mut rlp = RlpStream::new(); - rlp.append(&receipt); - return Some(rlp.out()); + return Some(receipt); } None } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index ebe70dcd82c..5b78a54b34e 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use itertools::Itertools; use block::{OpenBlock, SealedBlock, ClosedBlock}; -use blockchain::TreeRoute; +use blockchain::{BlockReceipts, TreeRoute}; use client::Mode; use encoded; use vm::LastHashes; @@ -282,7 +282,7 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra fn transaction_receipt(&self, id: TransactionId) -> Option; /// Get localized receipts for all transaction in given block. - fn block_receipts(&self, id: BlockId) -> Option>; + fn localized_block_receipts(&self, id: BlockId) -> Option>; /// Get a tree route between `from` and `to`. /// See `BlockChain::tree_route`. @@ -294,8 +294,8 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra /// Get latest state node fn state_data(&self, hash: &H256) -> Option; - /// Get raw block receipts data by block header hash. - fn encoded_block_receipts(&self, hash: &H256) -> Option; + /// Get block receipts data by block header hash. + fn block_receipts(&self, hash: &H256) -> Option; /// Get block queue information. fn queue_info(&self) -> BlockQueueInfo; diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index fefa9b5e051..d73a797773a 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -139,6 +139,9 @@ extern crate trace_time; #[cfg_attr(test, macro_use)] extern crate evm; +#[cfg(test)] +extern crate env_logger; + pub extern crate ethstore; #[macro_use] diff --git a/ethcore/src/snapshot/error.rs b/ethcore/src/snapshot/error.rs index 527b4e2882c..eb1a16278b0 100644 --- a/ethcore/src/snapshot/error.rs +++ b/ethcore/src/snapshot/error.rs @@ -65,6 +65,8 @@ pub enum Error { BadEpochProof(u64), /// Wrong chunk format. WrongChunkFormat(String), + /// Unlinked ancient block chain + UnlinkedAncientBlockChain, } impl fmt::Display for Error { @@ -91,6 +93,7 @@ impl fmt::Display for Error { Error::SnapshotsUnsupported => write!(f, "Snapshots unsupported by consensus engine."), Error::BadEpochProof(i) => write!(f, "Bad epoch proof for transition to epoch {}", i), Error::WrongChunkFormat(ref msg) => write!(f, "Wrong chunk format: {}", msg), + Error::UnlinkedAncientBlockChain => write!(f, "Unlinked ancient blocks chain"), } } } diff --git a/ethcore/src/snapshot/mod.rs b/ethcore/src/snapshot/mod.rs index fbe6d6a166b..bd7e3cb51dd 100644 --- a/ethcore/src/snapshot/mod.rs +++ b/ethcore/src/snapshot/mod.rs @@ -56,7 +56,7 @@ use rand::{Rng, OsRng}; pub use self::error::Error; pub use self::consensus::*; -pub use self::service::{Service, DatabaseRestore}; +pub use self::service::{SnapshotClient, Service, DatabaseRestore}; pub use self::traits::SnapshotService; pub use self::watcher::Watcher; pub use types::snapshot_manifest::ManifestData; diff --git a/ethcore/src/snapshot/service.rs b/ethcore/src/snapshot/service.rs index 97fc516b832..f547faab1e0 100644 --- a/ethcore/src/snapshot/service.rs +++ b/ethcore/src/snapshot/service.rs @@ -22,12 +22,13 @@ use std::fs::{self, File}; use std::path::PathBuf; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::cmp; use super::{ManifestData, StateRebuilder, Rebuilder, RestorationStatus, SnapshotService, MAX_CHUNK_SIZE}; use super::io::{SnapshotReader, LooseReader, SnapshotWriter, LooseWriter}; use blockchain::{BlockChain, BlockChainDB, BlockChainDBHandler}; -use client::{Client, ChainInfo, ClientIoMessage}; +use client::{BlockInfo, BlockChainClient, Client, ChainInfo, ClientIoMessage}; use engines::EthEngine; use error::{Error, ErrorKind as SnapshotErrorKind}; use snapshot::{Error as SnapshotError}; @@ -40,6 +41,7 @@ use ethereum_types::H256; use parking_lot::{Mutex, RwLock, RwLockReadGuard}; use bytes::Bytes; use journaldb::Algorithm; +use kvdb::DBTransaction; use snappy; /// Helper for removing directories in case of error. @@ -203,6 +205,9 @@ impl Restoration { /// Type alias for client io channel. pub type Channel = IoChannel; +/// Trait alias for the Client Service used +pub trait SnapshotClient: BlockChainClient + BlockInfo + DatabaseRestore {} + /// Snapshot service parameters. pub struct ServiceParams { /// The consensus engine this is built on. @@ -219,7 +224,7 @@ pub struct ServiceParams { /// Usually "/snapshot" pub snapshot_root: PathBuf, /// A handle for database restoration. - pub db_restore: Arc, + pub client: Arc, } /// `SnapshotService` implementation. @@ -236,7 +241,7 @@ pub struct Service { genesis_block: Bytes, state_chunks: AtomicUsize, block_chunks: AtomicUsize, - db_restore: Arc, + client: Arc, progress: super::Progress, taking_snapshot: AtomicBool, restoring_snapshot: AtomicBool, @@ -257,7 +262,7 @@ impl Service { genesis_block: params.genesis_block, state_chunks: AtomicUsize::new(0), block_chunks: AtomicUsize::new(0), - db_restore: params.db_restore, + client: params.client, progress: Default::default(), taking_snapshot: AtomicBool::new(false), restoring_snapshot: AtomicBool::new(false), @@ -334,12 +339,110 @@ impl Service { // replace one the client's database with our own. fn replace_client_db(&self) -> Result<(), Error> { - let our_db = self.restoration_db(); + let migrated_blocks = self.migrate_blocks()?; + trace!(target: "snapshot", "Migrated {} ancient blocks", migrated_blocks); - self.db_restore.restore_db(&*our_db.to_string_lossy())?; + let rest_db = self.restoration_db(); + self.client.restore_db(&*rest_db.to_string_lossy())?; Ok(()) } + // Migrate the blocks in the current DB into the new chain + fn migrate_blocks(&self) -> Result { + // Count the number of migrated blocks + let mut count = 0; + let rest_db = self.restoration_db(); + + let cur_chain_info = self.client.chain_info(); + + let next_db = self.restoration_db_handler.open(&rest_db)?; + let next_chain = BlockChain::new(Default::default(), &[], next_db.clone()); + let next_chain_info = next_chain.chain_info(); + + // The old database looks like this: + // [genesis, best_ancient_block] ... [first_block, best_block] + // If we are fully synced neither `best_ancient_block` nor `first_block` is set, and we can assume that the whole range from [genesis, best_block] is imported. + // The new database only contains the tip of the chain ([first_block, best_block]), + // so the useful set of blocks is defined as: + // [0 ... min(new.first_block, best_ancient_block or best_block)] + let find_range = || -> Option<(H256, H256)> { + let next_available_from = next_chain_info.first_block_number?; + let cur_available_to = cur_chain_info.ancient_block_number.unwrap_or(cur_chain_info.best_block_number); + + let highest_block_num = cmp::min(next_available_from.saturating_sub(1), cur_available_to); + + if highest_block_num == 0 { + return None; + } + + trace!(target: "snapshot", "Trying to import ancient blocks until {}", highest_block_num); + + // Here we start from the highest block number and go backward to 0, + // thus starting at `highest_block_num` and targetting `0`. + let target_hash = self.client.block_hash(BlockId::Number(0))?; + let start_hash = self.client.block_hash(BlockId::Number(highest_block_num))?; + + Some((start_hash, target_hash)) + }; + + let (start_hash, target_hash) = match find_range() { + Some(x) => x, + None => return Ok(0), + }; + + let mut batch = DBTransaction::new(); + let mut parent_hash = start_hash; + while parent_hash != target_hash { + // Early return if restoration is aborted + if !self.restoring_snapshot.load(Ordering::SeqCst) { + return Ok(count); + } + + let block = self.client.block(BlockId::Hash(parent_hash)).ok_or(::snapshot::error::Error::UnlinkedAncientBlockChain)?; + parent_hash = block.parent_hash(); + + let block_number = block.number(); + let block_receipts = self.client.block_receipts(&block.hash()); + let parent_total_difficulty = self.client.block_total_difficulty(BlockId::Hash(parent_hash)); + + match (block_receipts, parent_total_difficulty) { + (Some(block_receipts), Some(parent_total_difficulty)) => { + let block_receipts = block_receipts.receipts; + + next_chain.insert_unordered_block(&mut batch, block, block_receipts, Some(parent_total_difficulty), false, true); + count += 1; + }, + _ => break, + } + + // Writting changes to DB and logging every now and then + if block_number % 1_000 == 0 { + next_db.key_value().write_buffered(batch); + next_chain.commit(); + next_db.key_value().flush().expect("DB flush failed."); + batch = DBTransaction::new(); + } + + if block_number % 10_000 == 0 { + trace!(target: "snapshot", "Block restoration at #{}", block_number); + } + } + + // Final commit to the DB + next_db.key_value().write_buffered(batch); + next_chain.commit(); + next_db.key_value().flush().expect("DB flush failed."); + + // We couldn't reach the targeted hash + if parent_hash != target_hash { + return Err(::snapshot::error::Error::UnlinkedAncientBlockChain.into()); + } + + // Update best ancient block in the Next Chain + next_chain.update_best_ancient_block(&start_hash); + Ok(count) + } + /// Get a reference to the snapshot reader. pub fn reader(&self) -> RwLockReadGuard> { self.reader.read() @@ -480,12 +583,16 @@ impl Service { // Import previous chunks, continue if it fails self.import_prev_chunks(&mut res, manifest).ok(); - *self.status.lock() = RestorationStatus::Ongoing { - state_chunks: state_chunks as u32, - block_chunks: block_chunks as u32, - state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32, - block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32, - }; + // It could be that the restoration failed or completed in the meanwhile + let mut restoration_status = self.status.lock(); + if let RestorationStatus::Initializing { .. } = *restoration_status { + *restoration_status = RestorationStatus::Ongoing { + state_chunks: state_chunks as u32, + block_chunks: block_chunks as u32, + state_chunks_done: self.state_chunks.load(Ordering::SeqCst) as u32, + block_chunks_done: self.block_chunks.load(Ordering::SeqCst) as u32, + }; + } Ok(()) } @@ -752,26 +859,19 @@ impl Drop for Service { #[cfg(test)] mod tests { - use std::sync::Arc; use client::ClientIoMessage; use io::{IoService}; use spec::Spec; use journaldb::Algorithm; - use error::Error; use snapshot::{ManifestData, RestorationStatus, SnapshotService}; use super::*; use tempdir::TempDir; - use test_helpers::restoration_db_handler; - - struct NoopDBRestore; - impl DatabaseRestore for NoopDBRestore { - fn restore_db(&self, _new_db: &str) -> Result<(), Error> { - Ok(()) - } - } + use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; #[test] fn sends_async_messages() { + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices); let service = IoService::::start().unwrap(); let spec = Spec::new_test(); @@ -785,7 +885,7 @@ mod tests { pruning: Algorithm::Archive, channel: service.channel(), snapshot_root: dir, - db_restore: Arc::new(NoopDBRestore), + client: client, }; let service = Service::new(snapshot_params).unwrap(); diff --git a/ethcore/src/snapshot/tests/service.rs b/ethcore/src/snapshot/tests/service.rs index a5af63b01e2..d98543cbb73 100644 --- a/ethcore/src/snapshot/tests/service.rs +++ b/ethcore/src/snapshot/tests/service.rs @@ -16,26 +16,23 @@ //! Tests for the snapshot service. +use std::fs; use std::sync::Arc; use tempdir::TempDir; -use client::{Client, BlockInfo}; +use blockchain::BlockProvider; +use client::{Client, ClientConfig, ImportBlock, BlockInfo}; use ids::BlockId; +use snapshot::io::{PackedReader, PackedWriter, SnapshotReader, SnapshotWriter}; use snapshot::service::{Service, ServiceParams}; -use snapshot::{self, ManifestData, SnapshotService}; +use snapshot::{chunk_state, chunk_secondary, ManifestData, Progress, SnapshotService, RestorationStatus}; use spec::Spec; -use test_helpers::{generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use test_helpers::{new_db, new_temp_db, generate_dummy_client_with_spec_and_data, restoration_db_handler}; +use parking_lot::Mutex; use io::IoChannel; use kvdb_rocksdb::DatabaseConfig; - -struct NoopDBRestore; - -impl snapshot::DatabaseRestore for NoopDBRestore { - fn restore_db(&self, _new_db: &str) -> Result<(), ::error::Error> { - Ok(()) - } -} +use verification::queue::kind::blocks::Unverified; #[test] fn restored_is_equivalent() { @@ -46,7 +43,6 @@ fn restored_is_equivalent() { const TX_PER: usize = 5; let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; - let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, TX_PER, &gas_prices); let tempdir = TempDir::new("").unwrap(); @@ -73,7 +69,7 @@ fn restored_is_equivalent() { pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: path, - db_restore: client2.clone(), + client: client2.clone(), }; let service = Service::new(service_params).unwrap(); @@ -94,7 +90,7 @@ fn restored_is_equivalent() { service.feed_block_chunk(hash, &chunk); } - assert_eq!(service.status(), ::snapshot::RestorationStatus::Inactive); + assert_eq!(service.status(), RestorationStatus::Inactive); for x in 0..NUM_BLOCKS { let block1 = client.block(BlockId::Number(x as u64)).unwrap(); @@ -106,6 +102,9 @@ fn restored_is_equivalent() { #[test] fn guards_delete_folders() { + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, 400, 5, &gas_prices); + let spec = Spec::new_null(); let tempdir = TempDir::new("").unwrap(); let service_params = ServiceParams { @@ -115,7 +114,7 @@ fn guards_delete_folders() { pruning: ::journaldb::Algorithm::Archive, channel: IoChannel::disconnected(), snapshot_root: tempdir.path().to_owned(), - db_restore: Arc::new(NoopDBRestore), + client: client, }; let service = Service::new(service_params).unwrap(); @@ -146,3 +145,201 @@ fn guards_delete_folders() { assert!(!path.join("db").exists()); assert!(path.join("temp").exists()); } + +#[test] +fn keep_ancient_blocks() { + ::env_logger::init().ok(); + + // Test variables + const NUM_BLOCKS: u64 = 500; + const NUM_SNAPSHOT_BLOCKS: u64 = 300; + const SNAPSHOT_MODE: ::snapshot::PowSnapshot = ::snapshot::PowSnapshot { blocks: NUM_SNAPSHOT_BLOCKS, max_restore_blocks: NUM_SNAPSHOT_BLOCKS }; + + // Temporary folders + let tempdir = TempDir::new("").unwrap(); + let snapshot_path = tempdir.path().join("SNAP"); + + // Generate blocks + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let spec_f = Spec::new_null; + let spec = spec_f(); + let client = generate_dummy_client_with_spec_and_data(spec_f, NUM_BLOCKS as u32, 5, &gas_prices); + + let bc = client.chain(); + + // Create the Snapshot + let best_hash = bc.best_block_hash(); + let writer = Mutex::new(PackedWriter::new(&snapshot_path).unwrap()); + let block_hashes = chunk_secondary( + Box::new(SNAPSHOT_MODE), + &bc, + best_hash, + &writer, + &Progress::default() + ).unwrap(); + let state_db = client.state_db().journal_db().boxed_clone(); + let start_header = bc.block_header_data(&best_hash).unwrap(); + let state_root = start_header.state_root(); + let state_hashes = chunk_state( + state_db.as_hashdb(), + &state_root, + &writer, + &Progress::default(), + None + ).unwrap(); + + let manifest = ::snapshot::ManifestData { + version: 2, + state_hashes: state_hashes, + state_root: state_root, + block_hashes: block_hashes, + block_number: NUM_BLOCKS, + block_hash: best_hash, + }; + + writer.into_inner().finish(manifest.clone()).unwrap(); + + // Initialize the Client + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let client_db = new_temp_db(&tempdir.path()); + let client2 = Client::new( + ClientConfig::default(), + &spec, + client_db, + Arc::new(::miner::Miner::new_for_tests(&spec, None)), + IoChannel::disconnected(), + ).unwrap(); + + // Add some ancient blocks + for block_number in 1..50 { + let block_hash = bc.block_hash(block_number).unwrap(); + let block = bc.block(&block_hash).unwrap(); + client2.import_block(Unverified::from_rlp(block.into_inner()).unwrap()).unwrap(); + } + + client2.import_verified_blocks(); + client2.flush_queue(); + + // Restore the Snapshot + let reader = PackedReader::new(&snapshot_path).unwrap().unwrap(); + let service_params = ServiceParams { + engine: spec.engine.clone(), + genesis_block: spec.genesis_block(), + restoration_db_handler: restoration_db_handler(db_config), + pruning: ::journaldb::Algorithm::Archive, + channel: IoChannel::disconnected(), + snapshot_root: tempdir.path().to_owned(), + client: client2.clone(), + }; + let service = Service::new(service_params).unwrap(); + service.init_restore(manifest.clone(), false).unwrap(); + + for hash in &manifest.block_hashes { + let chunk = reader.chunk(*hash).unwrap(); + service.feed_block_chunk(*hash, &chunk); + } + + for hash in &manifest.state_hashes { + let chunk = reader.chunk(*hash).unwrap(); + service.feed_state_chunk(*hash, &chunk); + } + + match service.status() { + RestorationStatus::Inactive => (), + RestorationStatus::Failed => panic!("Snapshot Restoration has failed."), + RestorationStatus::Ongoing { .. } => panic!("Snapshot Restoration should be done."), + _ => panic!("Invalid Snapshot Service status."), + } + + // Check that the latest block number is the right one + assert_eq!(client2.block(BlockId::Latest).unwrap().number(), NUM_BLOCKS as u64); + + // Check that we have blocks in [NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1 ; NUM_BLOCKS] + // but none before + assert!(client2.block(BlockId::Number(NUM_BLOCKS - NUM_SNAPSHOT_BLOCKS + 1)).is_some()); + assert!(client2.block(BlockId::Number(100)).is_none()); + + // Check that the first 50 blocks have been migrated + for block_number in 1..49 { + assert!(client2.block(BlockId::Number(block_number)).is_some()); + } +} + +#[test] +fn recover_aborted_recovery() { + ::env_logger::init().ok(); + + const NUM_BLOCKS: u32 = 400; + let gas_prices = vec![1.into(), 2.into(), 3.into(), 999.into()]; + let client = generate_dummy_client_with_spec_and_data(Spec::new_null, NUM_BLOCKS, 5, &gas_prices); + + let spec = Spec::new_null(); + let tempdir = TempDir::new("").unwrap(); + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let client_db = new_db(); + let client2 = Client::new( + Default::default(), + &spec, + client_db, + Arc::new(::miner::Miner::new_for_tests(&spec, None)), + IoChannel::disconnected(), + ).unwrap(); + let service_params = ServiceParams { + engine: spec.engine.clone(), + genesis_block: spec.genesis_block(), + restoration_db_handler: restoration_db_handler(db_config), + pruning: ::journaldb::Algorithm::Archive, + channel: IoChannel::disconnected(), + snapshot_root: tempdir.path().to_owned(), + client: client2.clone(), + }; + + let service = Service::new(service_params).unwrap(); + service.take_snapshot(&client, NUM_BLOCKS as u64).unwrap(); + + let manifest = service.manifest().unwrap(); + service.init_restore(manifest.clone(), true).unwrap(); + + // Restore only the state chunks + for hash in &manifest.state_hashes { + let chunk = service.chunk(*hash).unwrap(); + service.feed_state_chunk(*hash, &chunk); + } + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32); + assert_eq!(block_chunks_done, 0); + }, + e => panic!("Snapshot restoration must be ongoing ; {:?}", e), + } + + // Abort the restore... + service.abort_restore(); + + // And try again! + service.init_restore(manifest.clone(), true).unwrap(); + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(state_chunks_done, manifest.state_hashes.len() as u32); + assert_eq!(block_chunks_done, 0); + }, + e => panic!("Snapshot restoration must be ongoing ; {:?}", e), + } + + // Remove the snapshot directory, and restart the restoration + // It shouldn't have restored any previous blocks + fs::remove_dir_all(tempdir.path()).unwrap(); + + // And try again! + service.init_restore(manifest.clone(), true).unwrap(); + + match service.status() { + RestorationStatus::Ongoing { block_chunks_done, state_chunks_done, .. } => { + assert_eq!(block_chunks_done, 0); + assert_eq!(state_chunks_done, 0); + }, + _ => panic!("Snapshot restoration must be ongoing"), + } +} diff --git a/ethcore/src/test_helpers.rs b/ethcore/src/test_helpers.rs index bf9ab020481..9f00f9662ce 100644 --- a/ethcore/src/test_helpers.rs +++ b/ethcore/src/test_helpers.rs @@ -41,7 +41,7 @@ use transaction::{Action, Transaction, SignedTransaction}; use views::BlockView; use blooms_db; use kvdb::KeyValueDB; -use kvdb_rocksdb; +use kvdb_rocksdb::{self, Database, DatabaseConfig}; use tempdir::TempDir; use verification::queue::kind::blocks::Unverified; use encoded; @@ -263,30 +263,30 @@ pub fn get_test_client_with_blocks(blocks: Vec) -> Arc { client } -/// Creates new test instance of `BlockChainDB` -pub fn new_db() -> Arc { - struct TestBlockChainDB { - _blooms_dir: TempDir, - _trace_blooms_dir: TempDir, - blooms: blooms_db::Database, - trace_blooms: blooms_db::Database, - key_value: Arc, - } +struct TestBlockChainDB { + _blooms_dir: TempDir, + _trace_blooms_dir: TempDir, + blooms: blooms_db::Database, + trace_blooms: blooms_db::Database, + key_value: Arc, +} - impl BlockChainDB for TestBlockChainDB { - fn key_value(&self) -> &Arc { - &self.key_value - } +impl BlockChainDB for TestBlockChainDB { + fn key_value(&self) -> &Arc { + &self.key_value + } - fn blooms(&self) -> &blooms_db::Database { - &self.blooms - } + fn blooms(&self) -> &blooms_db::Database { + &self.blooms + } - fn trace_blooms(&self) -> &blooms_db::Database { - &self.trace_blooms - } + fn trace_blooms(&self) -> &blooms_db::Database { + &self.trace_blooms } +} +/// Creates new test instance of `BlockChainDB` +pub fn new_db() -> Arc { let blooms_dir = TempDir::new("").unwrap(); let trace_blooms_dir = TempDir::new("").unwrap(); @@ -301,6 +301,26 @@ pub fn new_db() -> Arc { Arc::new(db) } +/// Creates a new temporary `BlockChainDB` on FS +pub fn new_temp_db(tempdir: &Path) -> Arc { + let blooms_dir = TempDir::new("").unwrap(); + let trace_blooms_dir = TempDir::new("").unwrap(); + let key_value_dir = tempdir.join("key_value"); + + let db_config = DatabaseConfig::with_columns(::db::NUM_COLUMNS); + let key_value_db = Database::open(&db_config, key_value_dir.to_str().unwrap()).unwrap(); + + let db = TestBlockChainDB { + blooms: blooms_db::Database::open(blooms_dir.path()).unwrap(), + trace_blooms: blooms_db::Database::open(trace_blooms_dir.path()).unwrap(), + _blooms_dir: blooms_dir, + _trace_blooms_dir: trace_blooms_dir, + key_value: Arc::new(key_value_db) + }; + + Arc::new(db) +} + /// Creates new instance of KeyValueDBHandler pub fn restoration_db_handler(config: kvdb_rocksdb::DatabaseConfig) -> Box { struct RestorationDBHandler { diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 1a0f99b921d..e0fc8ecddb6 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -970,6 +970,12 @@ impl ChainSync { self.state = SyncState::Blocks; self.continue_sync(io); }, + SyncState::SnapshotData => match io.snapshot_service().status() { + RestorationStatus::Inactive | RestorationStatus::Failed => { + self.state = SyncState::SnapshotWaiting; + }, + RestorationStatus::Initializing { .. } | RestorationStatus::Ongoing { .. } => (), + }, SyncState::SnapshotWaiting => { match io.snapshot_service().status() { RestorationStatus::Inactive => { diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 201e0d9f35a..4bce0ef9850 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -226,7 +226,8 @@ impl SyncSupplier { let mut added_receipts = 0usize; let mut data = Bytes::new(); for i in 0..count { - if let Some(mut receipts_bytes) = io.chain().encoded_block_receipts(&rlp.val_at::(i)?) { + if let Some(receipts) = io.chain().block_receipts(&rlp.val_at::(i)?) { + let mut receipts_bytes = ::rlp::encode(&receipts); data.append(&mut receipts_bytes); added_receipts += receipts_bytes.len(); added_headers += 1; diff --git a/rpc/src/v1/impls/parity.rs b/rpc/src/v1/impls/parity.rs index 8982b73e145..e973f7272de 100644 --- a/rpc/src/v1/impls/parity.rs +++ b/rpc/src/v1/impls/parity.rs @@ -434,7 +434,7 @@ impl Parity for ParityClient where BlockNumber::Earliest => BlockId::Earliest, BlockNumber::Latest => BlockId::Latest, }; - let receipts = try_bf!(self.client.block_receipts(id).ok_or_else(errors::unknown_block)); + let receipts = try_bf!(self.client.localized_block_receipts(id).ok_or_else(errors::unknown_block)); Box::new(future::ok(receipts.into_iter().map(Into::into).collect())) } From e2e79b2fa8e3ac726cf0b377614315bca8b35668 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Wed, 21 Nov 2018 20:11:01 +0100 Subject: [PATCH 04/18] Adjust requests costs for light client (#9925) * PIP Table Cost relative to average peers instead of max peers * Add tracing in PIP new_cost_table * Update stat peer_count * Use number of leeching peers for Light serve costs * Fix test::light_params_load_share_depends_on_max_peers (wrong type) * Remove (now) useless test * Remove `load_share` from LightParams.Config Prevent div. by 0 * Add LEECHER_COUNT_FACTOR * PR Grumble: u64 to u32 for f64 casting * Prevent u32 overflow for avg_peer_count * Add tests for LightSync::Statistics --- ethcore/light/src/net/mod.rs | 88 +++++++++++++++++++++++++++--- ethcore/light/src/net/tests/mod.rs | 34 +++++++++++- ethcore/sync/src/api.rs | 27 ++------- 3 files changed, 117 insertions(+), 32 deletions(-) diff --git a/ethcore/light/src/net/mod.rs b/ethcore/light/src/net/mod.rs index 590be038e21..cec2c699474 100644 --- a/ethcore/light/src/net/mod.rs +++ b/ethcore/light/src/net/mod.rs @@ -28,7 +28,7 @@ use parking_lot::{Mutex, RwLock}; use provider::Provider; use request::{Request, NetworkRequests as Requests, Response}; use rlp::{RlpStream, Rlp}; -use std::collections::{HashMap, HashSet}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::ops::{BitOr, BitAnd, Not}; use std::sync::Arc; @@ -38,7 +38,7 @@ use std::time::{Duration, Instant}; use self::request_credits::{Credits, FlowParams}; use self::context::{Ctx, TickCtx}; use self::error::Punishment; -use self::load_timer::{LoadDistribution, NullStore}; +use self::load_timer::{LoadDistribution, NullStore, MOVING_SAMPLE_SIZE}; use self::request_set::RequestSet; use self::id_guard::IdGuard; @@ -70,6 +70,16 @@ const PROPAGATE_TIMEOUT_INTERVAL: Duration = Duration::from_secs(5); const RECALCULATE_COSTS_TIMEOUT: TimerToken = 3; const RECALCULATE_COSTS_INTERVAL: Duration = Duration::from_secs(60 * 60); +const STATISTICS_TIMEOUT: TimerToken = 4; +const STATISTICS_INTERVAL: Duration = Duration::from_secs(15); + +/// Maximum load share for the light server +pub const MAX_LIGHTSERV_LOAD: f64 = 0.5; + +/// Factor to multiply leecher count to cater for +/// extra sudden connections (should be >= 1.0) +pub const LEECHER_COUNT_FACTOR: f64 = 1.25; + // minimum interval between updates. const UPDATE_INTERVAL: Duration = Duration::from_millis(5000); @@ -256,18 +266,18 @@ pub trait Handler: Send + Sync { pub struct Config { /// How many stored seconds of credits peers should be able to accumulate. pub max_stored_seconds: u64, - /// How much of the total load capacity each peer should be allowed to take. - pub load_share: f64, + /// The network config median peers (used as default peer count) + pub median_peers: f64, } impl Default for Config { fn default() -> Self { - const LOAD_SHARE: f64 = 1.0 / 25.0; + const MEDIAN_PEERS: f64 = 25.0; const MAX_ACCUMULATED: u64 = 60 * 5; // only charge for 5 minutes. Config { max_stored_seconds: MAX_ACCUMULATED, - load_share: LOAD_SHARE, + median_peers: MEDIAN_PEERS, } } } @@ -335,6 +345,42 @@ mod id_guard { } } +/// Provides various statistics that could +/// be used to compute costs +pub struct Statistics { + /// Samples of peer count + peer_counts: VecDeque, +} + +impl Statistics { + /// Create a new Statistics instance + pub fn new() -> Self { + Statistics { + peer_counts: VecDeque::with_capacity(MOVING_SAMPLE_SIZE), + } + } + + /// Add a new peer_count sample + pub fn add_peer_count(&mut self, peer_count: usize) { + while self.peer_counts.len() >= MOVING_SAMPLE_SIZE { + self.peer_counts.pop_front(); + } + self.peer_counts.push_back(peer_count); + } + + /// Get the average peer count from previous samples. Is always >= 1.0 + pub fn avg_peer_count(&self) -> f64 { + let len = self.peer_counts.len(); + if len == 0 { + return 1.0; + } + let avg = self.peer_counts.iter() + .fold(0, |sum: u32, &v| sum.saturating_add(v as u32)) as f64 + / len as f64; + avg.max(1.0) + } +} + /// This is an implementation of the light ethereum network protocol, abstracted /// over a `Provider` of data and a p2p network. /// @@ -359,6 +405,7 @@ pub struct LightProtocol { req_id: AtomicUsize, sample_store: Box, load_distribution: LoadDistribution, + statistics: RwLock, } impl LightProtocol { @@ -369,9 +416,11 @@ impl LightProtocol { let genesis_hash = provider.chain_info().genesis_hash; let sample_store = params.sample_store.unwrap_or_else(|| Box::new(NullStore)); let load_distribution = LoadDistribution::load(&*sample_store); + // Default load share relative to median peers + let load_share = MAX_LIGHTSERV_LOAD / params.config.median_peers; let flow_params = FlowParams::from_request_times( |kind| load_distribution.expected_time(kind), - params.config.load_share, + load_share, Duration::from_secs(params.config.max_stored_seconds), ); @@ -389,6 +438,7 @@ impl LightProtocol { req_id: AtomicUsize::new(0), sample_store, load_distribution, + statistics: RwLock::new(Statistics::new()), } } @@ -408,6 +458,16 @@ impl LightProtocol { ) } + /// Get the number of active light peers downloading from the + /// node + pub fn leecher_count(&self) -> usize { + let credit_limit = *self.flow_params.read().limit(); + // Count the number of peers that used some credit + self.peers.read().iter() + .filter(|(_, p)| p.lock().local_credits.current() < credit_limit) + .count() + } + /// Make a request to a peer. /// /// Fails on: nonexistent peer, network error, peer not server, @@ -772,12 +832,16 @@ impl LightProtocol { fn begin_new_cost_period(&self, io: &IoContext) { self.load_distribution.end_period(&*self.sample_store); + let avg_peer_count = self.statistics.read().avg_peer_count(); + // Load share relative to average peer count +LEECHER_COUNT_FACTOR% + let load_share = MAX_LIGHTSERV_LOAD / (avg_peer_count * LEECHER_COUNT_FACTOR); let new_params = Arc::new(FlowParams::from_request_times( |kind| self.load_distribution.expected_time(kind), - self.config.load_share, + load_share, Duration::from_secs(self.config.max_stored_seconds), )); *self.flow_params.write() = new_params.clone(); + trace!(target: "pip", "New cost period: avg_peers={} ; cost_table:{:?}", avg_peer_count, new_params.cost_table()); let peers = self.peers.read(); let now = Instant::now(); @@ -797,6 +861,11 @@ impl LightProtocol { peer_info.awaiting_acknowledge = Some((now, new_params.clone())); } } + + fn tick_statistics(&self) { + let leecher_count = self.leecher_count(); + self.statistics.write().add_peer_count(leecher_count); + } } impl LightProtocol { @@ -1099,6 +1168,8 @@ impl NetworkProtocolHandler for LightProtocol { .expect("Error registering sync timer."); io.register_timer(RECALCULATE_COSTS_TIMEOUT, RECALCULATE_COSTS_INTERVAL) .expect("Error registering request timer interval token."); + io.register_timer(STATISTICS_TIMEOUT, STATISTICS_INTERVAL) + .expect("Error registering statistics timer."); } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { @@ -1119,6 +1190,7 @@ impl NetworkProtocolHandler for LightProtocol { TICK_TIMEOUT => self.tick_handlers(&io), PROPAGATE_TIMEOUT => self.propagate_transactions(&io), RECALCULATE_COSTS_TIMEOUT => self.begin_new_cost_period(&io), + STATISTICS_TIMEOUT => self.tick_statistics(), _ => warn!(target: "pip", "received timeout on unknown token {}", timer), } } diff --git a/ethcore/light/src/net/tests/mod.rs b/ethcore/light/src/net/tests/mod.rs index d1939015c75..791315f5f20 100644 --- a/ethcore/light/src/net/tests/mod.rs +++ b/ethcore/light/src/net/tests/mod.rs @@ -22,9 +22,10 @@ use ethcore::client::{EachBlockWith, TestBlockChainClient}; use ethcore::encoded; use ethcore::ids::BlockId; use ethereum_types::{H256, U256, Address}; -use net::{LightProtocol, Params, packet, Peer}; +use net::{LightProtocol, Params, packet, Peer, Statistics}; use net::context::IoContext; use net::status::{Capabilities, Status}; +use net::load_timer::MOVING_SAMPLE_SIZE; use network::{PeerId, NodeId}; use provider::Provider; use request; @@ -780,3 +781,34 @@ fn get_transaction_index() { let expected = Expect::Respond(packet::RESPONSE, response); proto.handle_packet(&expected, 1, packet::REQUEST, &request_body); } + +#[test] +fn sync_statistics() { + let mut stats = Statistics::new(); + + // Empty set should return 1.0 + assert_eq!(stats.avg_peer_count(), 1.0); + + // Average < 1.0 should return 1.0 + stats.add_peer_count(0); + assert_eq!(stats.avg_peer_count(), 1.0); + + stats = Statistics::new(); + + const N: f64 = 50.0; + + for i in 1..(N as usize + 1) { + stats.add_peer_count(i); + } + + // Compute the average for the sum 1..N + assert_eq!(stats.avg_peer_count(), N * (N + 1.0) / 2.0 / N); + + for _ in 1..(MOVING_SAMPLE_SIZE + 1) { + stats.add_peer_count(40); + } + + // Test that it returns the average of the last + // `MOVING_SAMPLE_SIZE` values + assert_eq!(stats.avg_peer_count(), 40.0); +} diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 8063ebecb1e..4296a4c2d07 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -264,12 +264,10 @@ pub struct EthSync { fn light_params( network_id: u64, - max_peers: u32, + median_peers: f64, pruning_info: PruningInfo, sample_store: Option>, ) -> LightParams { - const MAX_LIGHTSERV_LOAD: f64 = 0.5; - let mut light_params = LightParams { network_id: network_id, config: Default::default(), @@ -282,9 +280,7 @@ fn light_params( sample_store: sample_store, }; - let max_peers = ::std::cmp::max(max_peers, 1); - light_params.config.load_share = MAX_LIGHTSERV_LOAD / max_peers as f64; - + light_params.config.median_peers = median_peers; light_params } @@ -301,9 +297,10 @@ impl EthSync { .map(|mut p| { p.push("request_timings"); light_net::FileStore(p) }) .map(|store| Box::new(store) as Box<_>); + let median_peers = (params.network_config.min_peers + params.network_config.max_peers) as f64 / 2.0; let light_params = light_params( params.config.network_id, - params.network_config.max_peers, + median_peers, pruning_info, sample_store, ); @@ -940,19 +937,3 @@ impl LightSyncProvider for LightSync { Default::default() // TODO } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn light_params_load_share_depends_on_max_peers() { - let pruning_info = PruningInfo { - earliest_chain: 0, - earliest_state: 0, - }; - let params1 = light_params(0, 10, pruning_info.clone(), None); - let params2 = light_params(0, 20, pruning_info, None); - assert!(params1.config.load_share > params2.config.load_share) - } -} From da286a9a7f4db280febcedd1bb05824737bcfd7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 21 Nov 2018 21:30:03 +0000 Subject: [PATCH 05/18] Fix empty steps (#9939) * Don't send empty step twice or empty step then block. * Perform basic validation of locally sealed blocks. * Don't include empty step twice. --- ethcore/src/client/client.rs | 22 ++- ethcore/src/engines/authority_round/mod.rs | 190 +++++++++++++++------ 2 files changed, 154 insertions(+), 58 deletions(-) diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index 8412881a22e..d5ae9d80f6a 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -2287,26 +2287,34 @@ impl ScheduleInfo for Client { impl ImportSealedBlock for Client { fn import_sealed_block(&self, block: SealedBlock) -> EthcoreResult { - let h = block.header().hash(); let start = Instant::now(); + let header = block.header().clone(); let route = { + // Do a super duper basic verification to detect potential bugs + if let Err(e) = self.engine.verify_block_basic(&header) { + self.importer.bad_blocks.report( + block.rlp_bytes(), + format!("Detected an issue with locally sealed block: {}", e), + ); + return Err(e.into()); + } + // scope for self.import_lock let _import_lock = self.importer.import_lock.lock(); trace_time!("import_sealed_block"); - let number = block.header().number(); let block_data = block.rlp_bytes(); - let header = block.header().clone(); let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self); - trace!(target: "client", "Imported sealed block #{} ({})", number, h); + trace!(target: "client", "Imported sealed block #{} ({})", header.number(), header.hash()); self.state_db.write().sync_cache(&route.enacted, &route.retracted, false); route }; + let h = header.hash(); let route = ChainRoute::from([route].as_ref()); self.importer.miner.chain_new_blocks( self, - &[h.clone()], + &[h], &[], route.enacted(), route.retracted(), @@ -2314,10 +2322,10 @@ impl ImportSealedBlock for Client { ); self.notify(|notify| { notify.new_blocks( - vec![h.clone()], + vec![h], vec![], route.clone(), - vec![h.clone()], + vec![h], vec![], start.elapsed(), ); diff --git a/ethcore/src/engines/authority_round/mod.rs b/ethcore/src/engines/authority_round/mod.rs index 1a54ccec1b6..bf404b476f9 100644 --- a/ethcore/src/engines/authority_round/mod.rs +++ b/ethcore/src/engines/authority_round/mod.rs @@ -16,8 +16,8 @@ //! A blockchain engine that supports a non-instant BFT proof-of-authority. -use std::collections::{BTreeMap, HashSet}; -use std::fmt; +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::{cmp, fmt}; use std::iter::FromIterator; use std::ops::Deref; use std::sync::atomic::{AtomicUsize, AtomicBool, Ordering as AtomicOrdering}; @@ -123,10 +123,10 @@ struct Step { } impl Step { - fn load(&self) -> usize { self.inner.load(AtomicOrdering::SeqCst) } + fn load(&self) -> u64 { self.inner.load(AtomicOrdering::SeqCst) as u64 } fn duration_remaining(&self) -> Duration { let now = unix_now(); - let expected_seconds = (self.load() as u64) + let expected_seconds = self.load() .checked_add(1) .and_then(|ctr| ctr.checked_mul(self.duration as u64)) .map(Duration::from_secs); @@ -162,8 +162,8 @@ impl Step { } } - fn check_future(&self, given: usize) -> Result<(), Option>> { - const REJECTED_STEP_DRIFT: usize = 4; + fn check_future(&self, given: u64) -> Result<(), Option>> { + const REJECTED_STEP_DRIFT: u64 = 4; // Verify if the step is correct. if given <= self.load() { @@ -182,8 +182,8 @@ impl Step { let d = self.duration as u64; Err(Some(OutOfBounds { min: None, - max: Some(d * current as u64), - found: d * given as u64, + max: Some(d * current), + found: d * given, })) } else { Ok(()) @@ -192,8 +192,8 @@ impl Step { } // Chain scoring: total weight is sqrt(U256::max_value())*height - step -fn calculate_score(parent_step: U256, current_step: U256, current_empty_steps: U256) -> U256 { - U256::from(U128::max_value()) + parent_step - current_step + current_empty_steps +fn calculate_score(parent_step: u64, current_step: u64, current_empty_steps: usize) -> U256 { + U256::from(U128::max_value()) + U256::from(parent_step) - U256::from(current_step) + U256::from(current_empty_steps) } struct EpochManager { @@ -284,13 +284,26 @@ impl EpochManager { /// A message broadcast by authorities when it's their turn to seal a block but there are no /// transactions. Other authorities accumulate these messages and later include them in the seal as /// proof. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq)] struct EmptyStep { signature: H520, - step: usize, + step: u64, parent_hash: H256, } +impl PartialOrd for EmptyStep { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for EmptyStep { + fn cmp(&self, other: &Self) -> cmp::Ordering { + self.step.cmp(&other.step) + .then_with(|| self.parent_hash.cmp(&other.parent_hash)) + .then_with(|| self.signature.cmp(&other.signature)) + } +} + impl EmptyStep { fn from_sealed(sealed_empty_step: SealedEmptyStep, parent_hash: &H256) -> EmptyStep { let signature = sealed_empty_step.signature; @@ -353,7 +366,7 @@ pub fn empty_step_full_rlp(signature: &H520, empty_step_rlp: &[u8]) -> Vec { s.out() } -pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec { +pub fn empty_step_rlp(step: u64, parent_hash: &H256) -> Vec { let mut s = RlpStream::new_list(2); s.append(&step).append(parent_hash); s.out() @@ -365,7 +378,7 @@ pub fn empty_step_rlp(step: usize, parent_hash: &H256) -> Vec { /// empty message is included. struct SealedEmptyStep { signature: H520, - step: usize, + step: u64, } impl Encodable for SealedEmptyStep { @@ -399,7 +412,7 @@ pub struct AuthorityRound { validators: Box, validate_score_transition: u64, validate_step_transition: u64, - empty_steps: Mutex>, + empty_steps: Mutex>, epoch_manager: Mutex, immediate_transitions: bool, block_reward: U256, @@ -494,7 +507,7 @@ fn header_expected_seal_fields(header: &Header, empty_steps_transition: u64) -> } } -fn header_step(header: &Header, empty_steps_transition: u64) -> Result { +fn header_step(header: &Header, empty_steps_transition: u64) -> Result { let expected_seal_fields = header_expected_seal_fields(header, empty_steps_transition); Rlp::new(&header.seal().get(0).expect( &format!("was either checked with verify_block_basic or is genesis; has {} fields; qed (Make sure the spec file has a correct genesis seal)", expected_seal_fields))).as_val() @@ -533,17 +546,17 @@ fn header_empty_steps_signers(header: &Header, empty_steps_transition: u64) -> R } } -fn step_proposer(validators: &ValidatorSet, bh: &H256, step: usize) -> Address { - let proposer = validators.get(bh, step); +fn step_proposer(validators: &ValidatorSet, bh: &H256, step: u64) -> Address { + let proposer = validators.get(bh, step as usize); trace!(target: "engine", "Fetched proposer for step {}: {}", step, proposer); proposer } -fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: usize, address: &Address) -> bool { +fn is_step_proposer(validators: &ValidatorSet, bh: &H256, step: u64, address: &Address) -> bool { step_proposer(validators, bh, step) == *address } -fn verify_timestamp(step: &Step, header_step: usize) -> Result<(), BlockError> { +fn verify_timestamp(step: &Step, header_step: u64) -> Result<(), BlockError> { match step.check_future(header_step) { Err(None) => { trace!(target: "engine", "verify_timestamp: block from the future"); @@ -564,7 +577,7 @@ fn verify_external(header: &Header, validators: &ValidatorSet, empty_steps_trans let header_step = header_step(header, empty_steps_transition)?; let proposer_signature = header_signature(header, empty_steps_transition)?; - let correct_proposer = validators.get(header.parent_hash(), header_step); + let correct_proposer = validators.get(header.parent_hash(), header_step as usize); let is_invalid_proposer = *header.author() != correct_proposer || { let empty_steps_rlp = if header.number() >= empty_steps_transition { Some(header_empty_steps_raw(header)) @@ -634,13 +647,13 @@ impl AuthorityRound { panic!("authority_round: step duration can't be zero") } let should_timeout = our_params.start_step.is_none(); - let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))) as usize; + let initial_step = our_params.start_step.unwrap_or_else(|| (unix_now().as_secs() / (our_params.step_duration as u64))); let engine = Arc::new( AuthorityRound { transition_service: IoService::<()>::start()?, step: Arc::new(PermissionedStep { inner: Step { - inner: AtomicUsize::new(initial_step), + inner: AtomicUsize::new(initial_step as usize), calibrate: our_params.start_step.is_none(), duration: our_params.step_duration, }, @@ -651,7 +664,7 @@ impl AuthorityRound { validators: our_params.validators, validate_score_transition: our_params.validate_score_transition, validate_step_transition: our_params.validate_step_transition, - empty_steps: Mutex::new(Vec::new()), + empty_steps: Default::default(), epoch_manager: Mutex::new(EpochManager::blank()), immediate_transitions: our_params.immediate_transitions, block_reward: our_params.block_reward, @@ -699,22 +712,41 @@ impl AuthorityRound { }) } - fn empty_steps(&self, from_step: U256, to_step: U256, parent_hash: H256) -> Vec { - self.empty_steps.lock().iter().filter(|e| { - U256::from(e.step) > from_step && - U256::from(e.step) < to_step && - e.parent_hash == parent_hash - }).cloned().collect() + fn empty_steps(&self, from_step: u64, to_step: u64, parent_hash: H256) -> Vec { + let from = EmptyStep { + step: from_step + 1, + parent_hash, + signature: Default::default(), + }; + let to = EmptyStep { + step: to_step, + parent_hash: Default::default(), + signature: Default::default(), + }; + + if from >= to { + return vec![]; + } + + self.empty_steps.lock() + .range(from..to) + .filter(|e| e.parent_hash == parent_hash) + .cloned() + .collect() } - fn clear_empty_steps(&self, step: U256) { + fn clear_empty_steps(&self, step: u64) { // clear old `empty_steps` messages - self.empty_steps.lock().retain(|e| U256::from(e.step) > step); + let mut empty_steps = self.empty_steps.lock(); + *empty_steps = empty_steps.split_off(&EmptyStep { + step: step + 1, + parent_hash: Default::default(), + signature: Default::default(), + }); } fn handle_empty_step_message(&self, empty_step: EmptyStep) { - let mut empty_steps = self.empty_steps.lock(); - empty_steps.push(empty_step); + self.empty_steps.lock().insert(empty_step); } fn generate_empty_step(&self, parent_hash: &H256) { @@ -744,7 +776,7 @@ impl AuthorityRound { } } - fn report_skipped(&self, header: &Header, current_step: usize, parent_step: usize, validators: &ValidatorSet, set_number: u64) { + fn report_skipped(&self, header: &Header, current_step: u64, parent_step: u64, validators: &ValidatorSet, set_number: u64) { // we're building on top of the genesis block so don't report any skipped steps if header.number() == 1 { return; @@ -937,12 +969,12 @@ impl Engine for AuthorityRound { let current_step = self.step.inner.load(); let current_empty_steps_len = if header.number() >= self.empty_steps_transition { - self.empty_steps(parent_step.into(), current_step.into(), parent.hash()).len() + self.empty_steps(parent_step, current_step, parent.hash()).len() } else { 0 }; - let score = calculate_score(parent_step.into(), current_step.into(), current_empty_steps_len.into()); + let score = calculate_score(parent_step, current_step, current_empty_steps_len); header.set_difficulty(score); } @@ -986,8 +1018,8 @@ impl Engine for AuthorityRound { } let header = block.header(); - let parent_step: U256 = header_step(parent, self.empty_steps_transition) - .expect("Header has been verified; qed").into(); + let parent_step = header_step(parent, self.empty_steps_transition) + .expect("Header has been verified; qed"); let step = self.step.inner.load(); @@ -1022,7 +1054,7 @@ impl Engine for AuthorityRound { if is_step_proposer(&*validators, header.parent_hash(), step, header.author()) { // this is guarded against by `can_propose` unless the block was signed // on the same step (implies same key) and on a different node. - if parent_step == step.into() { + if parent_step == step { warn!("Attempted to seal block on the same step as parent. Is this authority sealing with more than one node?"); return Seal::None; } @@ -1034,7 +1066,10 @@ impl Engine for AuthorityRound { block.transactions().is_empty() && empty_steps.len() < self.maximum_empty_steps { - self.generate_empty_step(header.parent_hash()); + if self.step.can_propose.compare_and_swap(true, false, AtomicOrdering::SeqCst) { + self.generate_empty_step(header.parent_hash()); + } + return Seal::None; } @@ -1058,7 +1093,7 @@ impl Engine for AuthorityRound { // report any skipped primaries between the parent block and // the block we're sealing, unless we have empty steps enabled if header.number() < self.empty_steps_transition { - self.report_skipped(header, step, u64::from(parent_step) as usize, &*validators, set_number); + self.report_skipped(header, step, parent_step, &*validators, set_number); } let mut fields = vec![ @@ -1593,12 +1628,12 @@ mod tests { // Two validators. // Spec starts with step 2. - header.set_difficulty(calculate_score(U256::from(0), U256::from(2), U256::zero())); + header.set_difficulty(calculate_score(0, 2, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&2usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); assert!(engine.verify_block_external(&header).is_err()); - header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero())); + header.set_difficulty(calculate_score(0, 1, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); @@ -1622,7 +1657,7 @@ mod tests { // Two validators. // Spec starts with step 2. - header.set_difficulty(calculate_score(U256::from(0), U256::from(1), U256::zero())); + header.set_difficulty(calculate_score(0, 1, 0)); let signature = tap.sign(addr, Some("0".into()), header.bare_hash()).unwrap(); header.set_seal(vec![encode(&1usize), encode(&(&*signature as &[u8]))]); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); @@ -1650,10 +1685,10 @@ mod tests { // Two validators. // Spec starts with step 2. header.set_seal(vec![encode(&5usize), encode(&(&*signature as &[u8]))]); - header.set_difficulty(calculate_score(U256::from(4), U256::from(5), U256::zero())); + header.set_difficulty(calculate_score(4, 5, 0)); assert!(engine.verify_block_family(&header, &parent_header).is_ok()); header.set_seal(vec![encode(&3usize), encode(&(&*signature as &[u8]))]); - header.set_difficulty(calculate_score(U256::from(4), U256::from(3), U256::zero())); + header.set_difficulty(calculate_score(4, 3, 0)); assert!(engine.verify_block_family(&header, &parent_header).is_err()); } @@ -1687,7 +1722,7 @@ mod tests { parent_header.set_seal(vec![encode(&1usize)]); parent_header.set_gas_limit("222222".parse::().unwrap()); let mut header: Header = Header::default(); - header.set_difficulty(calculate_score(U256::from(1), U256::from(3), U256::zero())); + header.set_difficulty(calculate_score(1, 3, 0)); header.set_gas_limit("222222".parse::().unwrap()); header.set_seal(vec![encode(&3usize)]); @@ -1801,14 +1836,14 @@ mod tests { (spec, tap, accounts) } - fn empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> EmptyStep { + fn empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> EmptyStep { let empty_step_rlp = super::empty_step_rlp(step, parent_hash); let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into(); let parent_hash = parent_hash.clone(); EmptyStep { step, signature, parent_hash } } - fn sealed_empty_step(engine: &EthEngine, step: usize, parent_hash: &H256) -> SealedEmptyStep { + fn sealed_empty_step(engine: &EthEngine, step: u64, parent_hash: &H256) -> SealedEmptyStep { let empty_step_rlp = super::empty_step_rlp(step, parent_hash); let signature = engine.sign(keccak(&empty_step_rlp)).unwrap().into(); SealedEmptyStep { signature, step } @@ -1844,6 +1879,11 @@ mod tests { // we've received the message assert!(notify.messages.read().contains(&empty_step_rlp)); + let len = notify.messages.read().len(); + + // make sure that we don't generate empty step for the second time + assert_eq!(engine.generate_seal(b1.block(), &genesis_header), Seal::None); + assert_eq!(len, notify.messages.read().len()); } #[test] @@ -2058,7 +2098,7 @@ mod tests { let empty_step3 = sealed_empty_step(engine, 3, &parent_header.hash()); let empty_steps = vec![empty_step2, empty_step3]; - header.set_difficulty(calculate_score(U256::from(0), U256::from(4), U256::from(2))); + header.set_difficulty(calculate_score(0, 4, 2)); let signature = tap.sign(addr1, Some("1".into()), header.bare_hash()).unwrap(); header.set_seal(vec![ encode(&4usize), @@ -2173,4 +2213,52 @@ mod tests { BTreeMap::default(), ); } + + #[test] + fn test_empty_steps() { + let last_benign = Arc::new(AtomicUsize::new(0)); + let params = AuthorityRoundParams { + step_duration: 4, + start_step: Some(1), + validators: Box::new(TestSet::new(Default::default(), last_benign.clone())), + validate_score_transition: 0, + validate_step_transition: 0, + immediate_transitions: true, + maximum_uncle_count_transition: 0, + maximum_uncle_count: 0, + empty_steps_transition: 0, + maximum_empty_steps: 10, + block_reward: Default::default(), + block_reward_contract_transition: 0, + block_reward_contract: Default::default(), + }; + + let mut c_params = ::spec::CommonParams::default(); + c_params.gas_limit_bound_divisor = 5.into(); + let machine = ::machine::EthereumMachine::regular(c_params, Default::default()); + let engine = AuthorityRound::new(params, machine).unwrap(); + + + let parent_hash: H256 = 1.into(); + let signature = H520::default(); + let step = |step: u64| EmptyStep { + step, + parent_hash, + signature, + }; + + engine.handle_empty_step_message(step(1)); + engine.handle_empty_step_message(step(3)); + engine.handle_empty_step_message(step(2)); + engine.handle_empty_step_message(step(1)); + + assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(1), step(2), step(3)]); + assert_eq!(engine.empty_steps(2, 3, parent_hash), vec![]); + assert_eq!(engine.empty_steps(2, 4, parent_hash), vec![step(3)]); + + engine.clear_empty_steps(2); + + assert_eq!(engine.empty_steps(0, 3, parent_hash), vec![]); + assert_eq!(engine.empty_steps(0, 4, parent_hash), vec![step(3)]); + } } From 87ccdaebde2f0419a4182a6dc51329a4f41719d6 Mon Sep 17 00:00:00 2001 From: Seun LanLege Date: Wed, 21 Nov 2018 22:41:23 +0100 Subject: [PATCH 06/18] prevent silent errors in daemon mode, closes #9367 (#9946) --- parity/run.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/parity/run.rs b/parity/run.rs index 03dbcaffb0b..7ee0250d6bd 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -404,11 +404,6 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: // create dirs used by parity cmd.dirs.create_dirs(cmd.acc_conf.unlocked_accounts.len() == 0, cmd.secretstore_conf.enabled)?; - // run in daemon mode - if let Some(pid_file) = cmd.daemon { - daemonize(pid_file)?; - } - //print out running parity environment print_running_environment(&spec.data_dir, &cmd.dirs, &db_dirs); @@ -801,6 +796,12 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: client.set_exit_handler(on_client_rq); updater.set_exit_handler(on_updater_rq); + // run in daemon mode + if let Some(pid_file) = cmd.daemon { + info!("Running as a daemon process!"); + daemonize(pid_file)?; + } + Ok(RunningClient { inner: RunningClientInner::Full { rpc: rpc_direct, From 41141c268790a8d0a3be0d32169fd66edd433c8a Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Sun, 25 Nov 2018 08:53:41 +0100 Subject: [PATCH 07/18] Fix a deadlock (#9952) * Update informant: - decimal in Mgas/s - print every 5s (not randomly between 5s and 10s) * Fix dead-lock in `blockchain.rs` * Update locks ordering --- ethcore/src/blockchain/blockchain.rs | 2 +- parity/informant.rs | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ethcore/src/blockchain/blockchain.rs b/ethcore/src/blockchain/blockchain.rs index e5af6420861..4286dc41482 100644 --- a/ethcore/src/blockchain/blockchain.rs +++ b/ethcore/src/blockchain/blockchain.rs @@ -1187,8 +1187,8 @@ impl BlockChain { let mut pending_block_details = self.pending_block_details.write(); let mut pending_write_txs = self.pending_transaction_addresses.write(); - let mut best_ancient_block = self.best_ancient_block.write(); let mut best_block = self.best_block.write(); + let mut best_ancient_block = self.best_ancient_block.write(); let mut write_block_details = self.block_details.write(); let mut write_hashes = self.block_hashes.write(); let mut write_txs = self.transaction_addresses.write(); diff --git a/parity/informant.rs b/parity/informant.rs index 9059786175a..4b4a5acdd84 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -256,16 +256,13 @@ impl Informant { } pub fn tick(&self) { - let elapsed = self.last_tick.read().elapsed(); - if elapsed < Duration::from_secs(5) { - return; - } + let now = Instant::now(); + let elapsed = now.duration_since(*self.last_tick.read()); let (client_report, full_report) = { let mut last_report = self.last_report.lock(); let full_report = self.target.report(); let diffed = full_report.client_report.clone() - &*last_report; - *last_report = full_report.client_report.clone(); (diffed, full_report) }; @@ -289,7 +286,8 @@ impl Informant { return; } - *self.last_tick.write() = Instant::now(); + *self.last_tick.write() = now; + *self.last_report.lock() = full_report.client_report.clone(); let paint = |c: Style, t: String| match self.with_color && atty::is(atty::Stream::Stdout) { true => format!("{}", c.paint(t)), @@ -306,7 +304,7 @@ impl Informant { format!("{} blk/s {} tx/s {} Mgas/s", paint(Yellow.bold(), format!("{:7.2}", (client_report.blocks_imported * 1000) as f64 / elapsed.as_milliseconds() as f64)), paint(Yellow.bold(), format!("{:6.1}", (client_report.transactions_applied * 1000) as f64 / elapsed.as_milliseconds() as f64)), - paint(Yellow.bold(), format!("{:4}", (client_report.gas_processed / (elapsed.as_milliseconds() * 1000)).low_u64())) + paint(Yellow.bold(), format!("{:6.1}", (client_report.gas_processed / 1000).low_u64() as f64 / elapsed.as_milliseconds() as f64)) ) } else { format!("{} hdr/s", From dedffe2d7cec9e569f7d167a2114ccb1ac7a5315 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 26 Nov 2018 12:05:02 +0100 Subject: [PATCH 08/18] Fix light client informant while syncing (#9932) * Add `is_idle` to LightSync to check importing status * Use SyncStateWrapper to make sure is_idle gets updates * Update is_major_import to use verified queue size as well * Add comment for `is_idle` * Add Debug to `SyncStateWrapper` * `fn get` -> `fn into_inner` --- ethcore/sync/src/light_sync/mod.rs | 126 ++++++++++++++++++----------- parity/informant.rs | 2 +- 2 files changed, 80 insertions(+), 48 deletions(-) diff --git a/ethcore/sync/src/light_sync/mod.rs b/ethcore/sync/src/light_sync/mod.rs index b4000c0822a..cda250f12ea 100644 --- a/ethcore/sync/src/light_sync/mod.rs +++ b/ethcore/sync/src/light_sync/mod.rs @@ -34,6 +34,7 @@ use std::collections::{HashMap, HashSet}; use std::mem; +use std::ops::Deref; use std::sync::Arc; use std::time::{Instant, Duration}; @@ -213,6 +214,44 @@ enum SyncState { Rounds(SyncRound), } +/// A wrapper around the SyncState that makes sure to +/// update the giving reference to `is_idle` +#[derive(Debug)] +struct SyncStateWrapper { + state: SyncState, +} + +impl SyncStateWrapper { + /// Create a new wrapper for SyncState::Idle + pub fn idle() -> Self { + SyncStateWrapper { + state: SyncState::Idle, + } + } + + /// Set the new state's value, making sure `is_idle` gets updated + pub fn set(&mut self, state: SyncState, is_idle_handle: &mut bool) { + *is_idle_handle = match state { + SyncState::Idle => true, + _ => false, + }; + self.state = state; + } + + /// Returns the internal state's value + pub fn into_inner(self) -> SyncState { + self.state + } +} + +impl Deref for SyncStateWrapper { + type Target = SyncState; + + fn deref(&self) -> &SyncState { + &self.state + } +} + struct ResponseCtx<'a> { peer: PeerId, req_id: ReqId, @@ -235,7 +274,9 @@ pub struct LightSync { pending_reqs: Mutex>, // requests from this handler client: Arc, rng: Mutex, - state: Mutex, + state: Mutex, + // We duplicate this state tracking to avoid deadlocks in `is_major_importing`. + is_idle: Mutex, } #[derive(Debug, Clone)] @@ -309,16 +350,17 @@ impl Handler for LightSync { if new_best.is_none() { debug!(target: "sync", "No peers remain. Reverting to idle"); - *self.state.lock() = SyncState::Idle; + self.set_state(&mut self.state.lock(), SyncState::Idle); } else { let mut state = self.state.lock(); - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.requests_abandoned(unfulfilled)), SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(unfulfilled)), }; + self.set_state(&mut state, next_state); } self.maintain_sync(ctx.as_basic()); @@ -390,12 +432,13 @@ impl Handler for LightSync { data: headers, }; - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.process_response(&ctx, &*self.client)), SyncState::Rounds(round) => SyncState::Rounds(round.process_response(&ctx)), }; + self.set_state(&mut state, next_state); } self.maintain_sync(ctx.as_basic()); @@ -408,12 +451,18 @@ impl Handler for LightSync { // private helpers impl LightSync { + /// Sets the LightSync's state, and update + /// `is_idle` + fn set_state(&self, state: &mut SyncStateWrapper, next_state: SyncState) { + state.set(next_state, &mut self.is_idle.lock()); + } + // Begins a search for the common ancestor and our best block. // does not lock state, instead has a mutable reference to it passed. - fn begin_search(&self, state: &mut SyncState) { + fn begin_search(&self, state: &mut SyncStateWrapper) { if let None = *self.best_seen.lock() { // no peers. - *state = SyncState::Idle; + self.set_state(state, SyncState::Idle); return; } @@ -422,7 +471,8 @@ impl LightSync { trace!(target: "sync", "Beginning search for common ancestor from {:?}", (chain_info.best_block_number, chain_info.best_block_hash)); - *state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number)); + let next_state = SyncState::AncestorSearch(AncestorSearch::begin(chain_info.best_block_number)); + self.set_state(state, next_state); } // handles request dispatch, block import, state machine transitions, and timeouts. @@ -435,7 +485,7 @@ impl LightSync { let chain_info = client.chain_info(); let mut state = self.state.lock(); - debug!(target: "sync", "Maintaining sync ({:?})", &*state); + debug!(target: "sync", "Maintaining sync ({:?})", **state); // drain any pending blocks into the queue. { @@ -445,11 +495,12 @@ impl LightSync { loop { if client.queue_info().is_full() { break } - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(round) => SyncState::Rounds(round.drain(&mut sink, Some(DRAIN_AMOUNT))), other => other, }; + self.set_state(&mut state, next_state); if sink.is_empty() { break } trace!(target: "sync", "Drained {} headers to import", sink.len()); @@ -483,15 +534,15 @@ impl LightSync { let network_score = other.as_ref().map(|target| target.head_td); trace!(target: "sync", "No target to sync to. Network score: {:?}, Local score: {:?}", network_score, best_td); - *state = SyncState::Idle; + self.set_state(&mut state, SyncState::Idle); return; } }; - match mem::replace(&mut *state, SyncState::Idle) { + match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(SyncRound::Abort(reason, remaining)) => { if remaining.len() > 0 { - *state = SyncState::Rounds(SyncRound::Abort(reason, remaining)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::Abort(reason, remaining))); return; } @@ -505,7 +556,7 @@ impl LightSync { AbortReason::NoResponses => {} AbortReason::TargetReached => { debug!(target: "sync", "Sync target reached. Going idle"); - *state = SyncState::Idle; + self.set_state(&mut state, SyncState::Idle); return; } } @@ -514,15 +565,15 @@ impl LightSync { self.begin_search(&mut state); } SyncState::AncestorSearch(AncestorSearch::FoundCommon(num, hash)) => { - *state = SyncState::Rounds(SyncRound::begin((num, hash), sync_target)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((num, hash), sync_target))); } SyncState::AncestorSearch(AncestorSearch::Genesis) => { // Same here. let g_hash = chain_info.genesis_hash; - *state = SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target)); + self.set_state(&mut state, SyncState::Rounds(SyncRound::begin((0, g_hash), sync_target))); } SyncState::Idle => self.begin_search(&mut state), - other => *state = other, // restore displaced state. + other => self.set_state(&mut state, other), // restore displaced state. } } @@ -543,12 +594,13 @@ impl LightSync { } drop(pending_reqs); - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Idle => SyncState::Idle, SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.requests_abandoned(&unfulfilled)), SyncState::Rounds(round) => SyncState::Rounds(round.requests_abandoned(&unfulfilled)), }; + self.set_state(&mut state, next_state); } } @@ -605,34 +657,14 @@ impl LightSync { None }; - *state = match mem::replace(&mut *state, SyncState::Idle) { + let next_state = match mem::replace(&mut *state, SyncStateWrapper::idle()).into_inner() { SyncState::Rounds(round) => SyncState::Rounds(round.dispatch_requests(dispatcher)), SyncState::AncestorSearch(search) => SyncState::AncestorSearch(search.dispatch_request(dispatcher)), other => other, }; - } - } - - fn is_major_importing_do_wait(&self, wait: bool) -> bool { - const EMPTY_QUEUE: usize = 3; - - if self.client.as_light_client().queue_info().unverified_queue_size > EMPTY_QUEUE { - return true; - } - let mg_state = if wait { - self.state.lock() - } else { - if let Some(mg_state) = self.state.try_lock() { - mg_state - } else { - return false; - } - }; - match *mg_state { - SyncState::Idle => false, - _ => true, + self.set_state(&mut state, next_state); } } } @@ -651,7 +683,8 @@ impl LightSync { pending_reqs: Mutex::new(HashMap::new()), client: client, rng: Mutex::new(OsRng::new()?), - state: Mutex::new(SyncState::Idle), + state: Mutex::new(SyncStateWrapper::idle()), + is_idle: Mutex::new(true), }) } } @@ -666,9 +699,6 @@ pub trait SyncInfo { /// Whether major sync is underway. fn is_major_importing(&self) -> bool; - - /// Whether major sync is underway, skipping some synchronization. - fn is_major_importing_no_sync(&self) -> bool; } impl SyncInfo for LightSync { @@ -681,11 +711,13 @@ impl SyncInfo for LightSync { } fn is_major_importing(&self) -> bool { - self.is_major_importing_do_wait(true) - } + const EMPTY_QUEUE: usize = 3; + + let queue_info = self.client.as_light_client().queue_info(); + let is_verifying = queue_info.unverified_queue_size + queue_info.verified_queue_size > EMPTY_QUEUE; + let is_syncing = !*self.is_idle.lock(); - fn is_major_importing_no_sync(&self) -> bool { - self.is_major_importing_do_wait(false) + is_verifying || is_syncing } } diff --git a/parity/informant.rs b/parity/informant.rs index 4b4a5acdd84..8cc37813ce2 100644 --- a/parity/informant.rs +++ b/parity/informant.rs @@ -184,7 +184,7 @@ impl InformantData for LightNodeInformantData { fn executes_transactions(&self) -> bool { false } fn is_major_importing(&self) -> bool { - self.sync.is_major_importing_no_sync() + self.sync.is_major_importing() } fn report(&self) -> Report { From 1ba2cb40a4dfe9cdfef449d23ee1387b142000ec Mon Sep 17 00:00:00 2001 From: Afri Schoedon <5chdn@users.noreply.github.com> Date: Tue, 27 Nov 2018 12:51:27 +0100 Subject: [PATCH 09/18] ci: rearrange pipeline by logic (#9970) * ci: rearrange pipeline by logic * ci: rename docs script --- .gitlab-ci.yml | 22 ++++++++++--------- .../{docs-jsonrpc.sh => publish-docs.sh} | 0 2 files changed, 12 insertions(+), 10 deletions(-) rename scripts/gitlab/{docs-jsonrpc.sh => publish-docs.sh} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bf11cad67c5..bb712e70e59 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,13 @@ test-linux: tags: - rust-stable +test-audit: + stage: test + script: + - scripts/gitlab/cargo-audit.sh + tags: + - rust-stable + build-linux: stage: build only: *releaseable_branches @@ -104,25 +111,18 @@ publish-awss3: tags: - shell -docs-jsonrpc: - stage: optional +publish-docs: + stage: publish only: - tags except: - nightly cache: {} script: - - scripts/gitlab/docs-jsonrpc.sh + - scripts/gitlab/publish-docs.sh tags: - shell -cargo-audit: - stage: optional - script: - - scripts/gitlab/cargo-audit.sh - tags: - - rust-stable - build-android: stage: optional image: parity/rust-android:gitlab-ci @@ -132,6 +132,7 @@ build-android: - scripts/gitlab/build-unix.sh tags: - rust-arm + allow_failure: true test-beta: stage: optional @@ -141,6 +142,7 @@ test-beta: - scripts/gitlab/test-all.sh beta tags: - rust-beta + allow_failure: true test-nightly: stage: optional diff --git a/scripts/gitlab/docs-jsonrpc.sh b/scripts/gitlab/publish-docs.sh similarity index 100% rename from scripts/gitlab/docs-jsonrpc.sh rename to scripts/gitlab/publish-docs.sh From f83993b3b06143a5be6152d0d6f3b26ef9c86fde Mon Sep 17 00:00:00 2001 From: gabriel klawitter Date: Tue, 27 Nov 2018 12:51:42 +0100 Subject: [PATCH 10/18] fix docker build (#9971) --- scripts/docker/hub/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/docker/hub/Dockerfile b/scripts/docker/hub/Dockerfile index 1ac92e71bf4..e9813754bbe 100644 --- a/scripts/docker/hub/Dockerfile +++ b/scripts/docker/hub/Dockerfile @@ -18,7 +18,6 @@ RUN rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* RUN groupadd -g 1000 parity \ && useradd -m -u 1000 -g parity -s /bin/sh parity -USER parity WORKDIR /home/parity @@ -33,6 +32,9 @@ RUN chmod +x ./entrypoint.sh COPY scripts/docker/hub/check_sync.sh /check_sync.sh +# switch to user parity here +USER parity + # setup ENTRYPOINT EXPOSE 5001 8080 8082 8083 8545 8546 8180 30303/tcp 30303/udp ENTRYPOINT ["./entrypoint.sh"] From 85781889388a72edd9c6aa5355da9c08a2b01924 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Wed, 28 Nov 2018 06:21:31 +0800 Subject: [PATCH 11/18] Deny unknown fields for chainspec (#9972) * Add deny_unknown_fields to chainspec * Add tests and fix existing one * Remove serde_ignored dependency for chainspec * Fix rpc test eth chain spec * Fix starting_nonce_test spec --- Cargo.lock | 248 +++++++++++++++++++++++++------ chainspec/Cargo.toml | 1 - chainspec/src/main.rs | 17 +-- json/src/spec/account.rs | 1 + json/src/spec/authority_round.rs | 2 + json/src/spec/basic_authority.rs | 2 + json/src/spec/builtin.rs | 5 + json/src/spec/engine.rs | 2 + json/src/spec/ethash.rs | 3 + json/src/spec/genesis.rs | 1 + json/src/spec/hardcoded_sync.rs | 3 +- json/src/spec/instant_seal.rs | 2 + json/src/spec/null_engine.rs | 2 + json/src/spec/params.rs | 1 + json/src/spec/seal.rs | 4 + json/src/spec/spec.rs | 67 ++++++++- json/src/spec/state.rs | 1 + json/src/spec/tendermint.rs | 2 + json/src/spec/validator_set.rs | 1 + rpc/src/v1/tests/eth.rs | 2 +- 20 files changed, 305 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dfd911b2593..4a7c03391a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,6 +185,11 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "cast" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cc" version = "1.0.25" @@ -205,7 +210,6 @@ name = "chainspec" version = "0.1.0" dependencies = [ "ethjson 0.1.0", - "serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -285,6 +289,51 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "criterion" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion-plot 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion-stats 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", + "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "handlebars 0.32.4 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools-num 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "simplelog 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "walkdir 2.2.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "criterion-plot" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "criterion-stats" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "crossbeam" version = "0.3.2" @@ -358,6 +407,23 @@ name = "crunchy" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "csv" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "csv-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ct-logs" version = "0.4.0" @@ -409,14 +475,14 @@ dependencies = [ [[package]] name = "docopt" -version = "0.8.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -431,7 +497,6 @@ dependencies = [ "ethabi 6.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -492,10 +557,11 @@ dependencies = [ [[package]] name = "eth-secp256k1" version = "0.5.7" -source = "git+https://github.com/paritytech/rust-secp256k1#db81cfea59014b4d176f10f86ed52e1a130b6822" +source = "git+https://github.com/paritytech/rust-secp256k1#ccc06e7480148b723eb44ac56cf4d20eec380b6f" dependencies = [ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -534,6 +600,7 @@ dependencies = [ name = "ethash" version = "1.12.0" dependencies = [ + "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -566,6 +633,7 @@ dependencies = [ "bn 0.4.4 (git+https://github.com/paritytech/bn)", "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "common-types 0.1.0", + "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -616,6 +684,8 @@ dependencies = [ "rlp_compress 0.1.0", "rlp_derive 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", "stop-guard 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -704,7 +774,7 @@ dependencies = [ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -849,6 +919,7 @@ dependencies = [ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parity-runtime 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1003,7 +1074,7 @@ dependencies = [ name = "ethkey-cli" version = "0.1.0" dependencies = [ - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethkey 0.3.0", "panic_hook 0.1.0", @@ -1044,7 +1115,7 @@ name = "ethstore-cli" version = "0.1.0" dependencies = [ "dir 0.1.2", - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethstore 0.2.0", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", @@ -1060,6 +1131,7 @@ name = "evm" version = "0.1.0" dependencies = [ "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1076,7 +1148,7 @@ dependencies = [ name = "evmbin" version = "0.1.0" dependencies = [ - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.12.0", "ethcore-transaction 0.1.0", @@ -1289,6 +1361,21 @@ name = "hamming" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "handlebars" +version = "0.32.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hardware-wallet" version = "1.12.0" @@ -1516,6 +1603,14 @@ dependencies = [ "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "itertools-num" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "itoa" version = "0.4.3" @@ -1562,7 +1657,7 @@ dependencies = [ [[package]] name = "jsonrpc-core" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "futures 0.1.25 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1574,7 +1669,7 @@ dependencies = [ [[package]] name = "jsonrpc-http-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "hyper 0.12.11 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1587,7 +1682,7 @@ dependencies = [ [[package]] name = "jsonrpc-ipc-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-server-utils 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1600,7 +1695,7 @@ dependencies = [ [[package]] name = "jsonrpc-macros" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-pubsub 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1610,7 +1705,7 @@ dependencies = [ [[package]] name = "jsonrpc-pubsub" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1620,7 +1715,7 @@ dependencies = [ [[package]] name = "jsonrpc-server-utils" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1636,7 +1731,7 @@ dependencies = [ [[package]] name = "jsonrpc-tcp-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", "jsonrpc-server-utils 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -1648,7 +1743,7 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" version = "9.0.0" -source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#207a277b098943864ecaf22dbab7a5e309866d6b" +source = "git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2#f8a54f46f7f1d68b4e7899ca1e929803bf966a5b" dependencies = [ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 9.0.0 (git+https://github.com/paritytech/jsonrpc.git?branch=parity-2.2)", @@ -2058,7 +2153,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2162,7 +2257,7 @@ version = "1.12.0" dependencies = [ "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", - "parity-ethereum 2.2.2", + "parity-ethereum 2.3.0", ] [[package]] @@ -2178,7 +2273,7 @@ dependencies = [ [[package]] name = "parity-ethereum" -version = "2.2.2" +version = "2.3.0" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2187,7 +2282,7 @@ dependencies = [ "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)", "daemonize 0.2.3 (git+https://github.com/paritytech/daemonize)", "dir 0.1.2", - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.12.0", "ethcore-io 1.12.0", @@ -2227,11 +2322,11 @@ dependencies = [ "parity-rpc-client 1.4.0", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.2", + "parity-version 2.3.0", "parity-whisper 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rpassword 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2382,7 +2477,7 @@ dependencies = [ "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.2.2", + "parity-version 2.3.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2479,7 +2574,7 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.12.0", "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-version 2.2.2", + "parity-version 2.3.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2489,7 +2584,7 @@ dependencies = [ [[package]] name = "parity-version" -version = "2.2.2" +version = "2.3.0" dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2601,6 +2696,21 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "pest" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pest_derive" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "petgraph" version = "0.4.13" @@ -2768,6 +2878,11 @@ name = "quick-error" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "quote" version = "0.6.8" @@ -3117,14 +3232,6 @@ dependencies = [ "syn 0.15.11 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "serde_ignored" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "serde_json" version = "1.0.32" @@ -3165,6 +3272,16 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "simplelog" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "term 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "siphasher" version = "0.1.3" @@ -3241,13 +3358,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strsim" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "strsim" -version = "0.7.0" +name = "syn" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "syn" @@ -3259,6 +3381,14 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "synstructure" version = "0.10.1" @@ -3284,6 +3414,15 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "term" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "term_size" version = "0.3.1" @@ -3338,6 +3477,11 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "thread-scoped" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "thread_local" version = "0.3.6" @@ -3752,6 +3896,11 @@ name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "unicode-xid" version = "0.1.0" @@ -3935,7 +4084,7 @@ dependencies = [ name = "whisper-cli" version = "0.1.0" dependencies = [ - "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-logger 1.12.0", "ethcore-network 1.12.0", "ethcore-network-devp2p 1.12.0", @@ -4066,6 +4215,7 @@ dependencies = [ "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" "checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62" +"checksum cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" "checksum cesu8 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" @@ -4075,6 +4225,9 @@ dependencies = [ "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a" "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc" +"checksum criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c47d2b548c5647e1a436dc0cb78d4ebf51b6bf7ab101ed76662828bdd4d3a24a" +"checksum criterion-plot 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6e649d6aacdbbdb94ec659561a309a71336fc5655ed408f3afd28df2fc0c4f4f" +"checksum criterion-stats 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ff43cac80562f91ead0b617c1be74edf350adfaa195809d355de98dfc8f9237d" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" "checksum crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3486aefc4c0487b9cb52372c97df0a48b8c249514af1ee99703bf70d2f2ceda1" @@ -4084,12 +4237,14 @@ dependencies = [ "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" "checksum crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c240f247c278fa08a6d4820a6a222bfc6e0d999e51ba67be94f44c905b2161f2" +"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452" +"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105" "checksum ct-logs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95a4bf5107667e12bf6ce31a3a5066d67acc88942b6742117a41198734aaccaa" "checksum ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)" = "" "checksum daemonize 0.2.3 (git+https://github.com/paritytech/daemonize)" = "" "checksum difference 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3304d19798a8e067e48d8e69b2c37f0b5e9b4e462504ad9e27e9f3fce02bba8" "checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" -"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" +"checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" "checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" @@ -4122,6 +4277,7 @@ dependencies = [ "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" +"checksum handlebars 0.32.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d89ec99d1594f285d4590fc32bac5f75cdab383f1123d504d27862c644a807dd" "checksum hashdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d91261ee336dd046ac7df28306cb297b7a7228bd1ae25e9a57f4ed5e0ab628c7" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" @@ -4144,6 +4300,7 @@ dependencies = [ "checksum ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)" = "70783119ac90828aaba91eae39db32c6c1b8838deea3637e5238efa0130801ab" "checksum itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4833d6978da405305126af4ac88569b5d71ff758581ce5a987dbfa3755f694fc" "checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" +"checksum itertools-num 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "83ca7b70b838f2e34bc6c2f367a1ed1cfe34fb82464adecadd31cdcc7da882fc" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1ecfa3b81afc64d9a6539c4eece96ac9a93c551c713a313800dade8e33d7b5c1" "checksum jni-sys 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" @@ -4220,6 +4377,8 @@ dependencies = [ "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" "checksum patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10438ba40c2f6e9ceca55277d8e7f6a5dafd58cabd802e6d97e16f02aab83a03" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" +"checksum pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" "checksum phf 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "cec29da322b242f4c3098852c77a0ca261c9c01b806cae85a5572a1eb94db9a6" "checksum phf_codegen 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "7d187f00cd98d5afbcd8898f6cf181743a449162aeb329dcd2f3849009e605ad" @@ -4237,6 +4396,7 @@ dependencies = [ "checksum pulldown-cmark 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8361e81576d2e02643b04950e487ec172b687180da65c731c03cf336784e6c07" "checksum pwasm-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "90d2b3c5bf24275fc77db6b14ec00a7a085d8ff9d1c4215fb6f6263e8d7b01bc" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" @@ -4275,12 +4435,12 @@ dependencies = [ "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" "checksum serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "15c141fc7027dd265a47c090bf864cf62b42c4d228bbcf4e51a0c9e2b0d3f7ef" "checksum serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)" = "225de307c6302bec3898c51ca302fc94a7a1697ef0845fcee6448f33c032249c" -"checksum serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142" "checksum serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "43344e7ce05d0d8280c5940cabb4964bea626aa58b1ec0e8c73fa2a8512a38ce" "checksum sha1 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "cc30b1e1e8c40c121ca33b86c23308a090d19974ef001b4bf6e61fd1a0fb095c" "checksum sha1 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "171698ce4ec7cbb93babeb3190021b4d72e96ccb98e33d277ae4ea959d6f2d9e" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" "checksum shell32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ee04b46101f57121c9da2b151988283b6beb79b34f5bb29a58ee48cb695122c" +"checksum simplelog 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e95345f185d5adeb8ec93459d2dc99654e294cc6ccf5b75414d8ea262de9a13" "checksum siphasher 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "833011ca526bd88f16778d32c699d325a9ad302fa06381cd66f7be63351d3f6d" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum skeptic 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24ebf8a06f5f8bae61ae5bbc7af7aac4ef6907ae975130faba1199e5fe82256a" @@ -4291,18 +4451,21 @@ dependencies = [ "checksum socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c4d11a52082057d87cb5caa31ad812f4504b97ab44732cd8359df2e9ff9f48e7" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00caf261d6f90f588f8450b8e1230fa0d5be49ee6140fdfbcb55335aff350970" -"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" +"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.15.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b036b7b35e846707c0e55c2c9441fa47867c0f87fca416921db3261b1d8c741a" +"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +"checksum term 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e6b677dd1e8214ea1ef4297f85dbcbed8e8cdddb561040cc998ca2551c37561" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" +"checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" @@ -4345,6 +4508,7 @@ dependencies = [ "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" +"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "55cd1f4b4e96b46aeb8d4855db4a7a9bd96eeeb5c6a1ab54593328761642ce2f" diff --git a/chainspec/Cargo.toml b/chainspec/Cargo.toml index 780fcdff446..8739ca7dcba 100644 --- a/chainspec/Cargo.toml +++ b/chainspec/Cargo.toml @@ -6,4 +6,3 @@ authors = ["Marek Kotewicz "] [dependencies] ethjson = { path = "../json" } serde_json = "1.0" -serde_ignored = "0.0.4" diff --git a/chainspec/src/main.rs b/chainspec/src/main.rs index 708d74b503d..43f9c805754 100644 --- a/chainspec/src/main.rs +++ b/chainspec/src/main.rs @@ -15,10 +15,8 @@ // along with Parity. If not, see . extern crate serde_json; -extern crate serde_ignored; extern crate ethjson; -use std::collections::BTreeSet; use std::{fs, env, process}; use ethjson::spec::Spec; @@ -41,24 +39,11 @@ fn main() { Err(_) => quit(&format!("{} could not be opened", path)), }; - let mut unused = BTreeSet::new(); - let mut deserializer = serde_json::Deserializer::from_reader(file); - - let spec: Result = serde_ignored::deserialize(&mut deserializer, |field| { - unused.insert(field.to_string()); - }); + let spec: Result = serde_json::from_reader(file); if let Err(err) = spec { quit(&format!("{} {}", path, err.to_string())); } - if !unused.is_empty() { - let err = unused.into_iter() - .map(|field| format!("{} unexpected field `{}`", path, field)) - .collect::>() - .join("\n"); - quit(&err); - } - println!("{} is valid", path); } diff --git a/json/src/spec/account.rs b/json/src/spec/account.rs index acc6d96b589..9a593db0cb2 100644 --- a/json/src/spec/account.rs +++ b/json/src/spec/account.rs @@ -23,6 +23,7 @@ use spec::builtin::Builtin; /// Spec account. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Account { /// Builtin contract. pub builtin: Option, diff --git a/json/src/spec/authority_round.rs b/json/src/spec/authority_round.rs index b4fcf4d78cb..17f70dcdd7f 100644 --- a/json/src/spec/authority_round.rs +++ b/json/src/spec/authority_round.rs @@ -23,6 +23,7 @@ use super::ValidatorSet; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRoundParams { /// Block duration, in seconds. #[serde(rename="stepDuration")] @@ -71,6 +72,7 @@ pub struct AuthorityRoundParams { /// Authority engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRound { /// Ethash params. pub params: AuthorityRoundParams, diff --git a/json/src/spec/basic_authority.rs b/json/src/spec/basic_authority.rs index 1e5c6b84561..d9b87336658 100644 --- a/json/src/spec/basic_authority.rs +++ b/json/src/spec/basic_authority.rs @@ -21,6 +21,7 @@ use super::ValidatorSet; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BasicAuthorityParams { /// Block duration. #[serde(rename="durationLimit")] @@ -31,6 +32,7 @@ pub struct BasicAuthorityParams { /// Authority engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct BasicAuthority { /// Ethash params. pub params: BasicAuthorityParams, diff --git a/json/src/spec/builtin.rs b/json/src/spec/builtin.rs index 850867d0950..df889a37719 100644 --- a/json/src/spec/builtin.rs +++ b/json/src/spec/builtin.rs @@ -20,6 +20,7 @@ use uint::Uint; /// Linear pricing. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Linear { /// Base price. pub base: usize, @@ -29,6 +30,7 @@ pub struct Linear { /// Pricing for modular exponentiation. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Modexp { /// Price divisor. pub divisor: usize, @@ -36,6 +38,7 @@ pub struct Modexp { /// Pricing for alt_bn128_pairing. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct AltBn128Pairing { /// Base price. pub base: usize, @@ -45,6 +48,7 @@ pub struct AltBn128Pairing { /// Pricing variants. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub enum Pricing { /// Linear pricing. #[serde(rename="linear")] @@ -59,6 +63,7 @@ pub enum Pricing { /// Spec builtin. #[derive(Debug, PartialEq, Deserialize, Clone)] +#[serde(deny_unknown_fields)] pub struct Builtin { /// Builtin name. pub name: String, diff --git a/json/src/spec/engine.rs b/json/src/spec/engine.rs index 020cc1fd0ba..4dec7324cb5 100644 --- a/json/src/spec/engine.rs +++ b/json/src/spec/engine.rs @@ -20,6 +20,7 @@ use super::{Ethash, BasicAuthority, AuthorityRound, Tendermint, NullEngine, Inst /// Engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum Engine { /// Null engine. #[serde(rename="null")] @@ -28,6 +29,7 @@ pub enum Engine { #[serde(rename="instantSeal")] InstantSeal(Option), /// Ethash engine. + #[serde(rename = "Ethash")] Ethash(Ethash), /// BasicAuthority engine. #[serde(rename="basicAuthority")] diff --git a/json/src/spec/ethash.rs b/json/src/spec/ethash.rs index 82cc986b20c..f73bce969e0 100644 --- a/json/src/spec/ethash.rs +++ b/json/src/spec/ethash.rs @@ -23,6 +23,7 @@ use hash::Address; /// Deserializable doppelganger of block rewards for EthashParams #[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] #[serde(untagged)] pub enum BlockReward { Single(Uint), @@ -31,6 +32,7 @@ pub enum BlockReward { /// Deserializable doppelganger of EthashParams. #[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct EthashParams { /// See main EthashParams docs. #[serde(rename="minimumDifficulty")] @@ -119,6 +121,7 @@ pub struct EthashParams { /// Ethash engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Ethash { /// Ethash params. pub params: EthashParams, diff --git a/json/src/spec/genesis.rs b/json/src/spec/genesis.rs index d8e2ad53579..5ec86eb26ee 100644 --- a/json/src/spec/genesis.rs +++ b/json/src/spec/genesis.rs @@ -23,6 +23,7 @@ use spec::Seal; /// Spec genesis. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Genesis { /// Seal. pub seal: Seal, diff --git a/json/src/spec/hardcoded_sync.rs b/json/src/spec/hardcoded_sync.rs index 8b00b5413b5..ab671994dc7 100644 --- a/json/src/spec/hardcoded_sync.rs +++ b/json/src/spec/hardcoded_sync.rs @@ -21,6 +21,7 @@ use uint::Uint; /// Spec hardcoded sync. #[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct HardcodedSync { /// Hexadecimal of the RLP encoding of the header of the block to start synchronization from. pub header: String, @@ -28,7 +29,7 @@ pub struct HardcodedSync { #[serde(rename="totalDifficulty")] pub total_difficulty: Uint, /// Ordered trie roots of blocks before and including `header`. - #[serde(rename="CHTs")] + #[serde(rename = "CHTs")] pub chts: Vec, } diff --git a/json/src/spec/instant_seal.rs b/json/src/spec/instant_seal.rs index 4621a19dd27..671f99c7bf4 100644 --- a/json/src/spec/instant_seal.rs +++ b/json/src/spec/instant_seal.rs @@ -18,6 +18,7 @@ /// Instant seal engine params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct InstantSealParams { /// Whether to enable millisecond timestamp. #[serde(rename="millisecondTimestamp")] @@ -27,6 +28,7 @@ pub struct InstantSealParams { /// Instant seal engine descriptor. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct InstantSeal { /// Instant seal parameters. pub params: InstantSealParams, diff --git a/json/src/spec/null_engine.rs b/json/src/spec/null_engine.rs index 87827bd5b9f..061c25aacbf 100644 --- a/json/src/spec/null_engine.rs +++ b/json/src/spec/null_engine.rs @@ -20,6 +20,7 @@ use uint::Uint; /// Authority params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct NullEngineParams { /// Block reward. #[serde(rename="blockReward")] @@ -28,6 +29,7 @@ pub struct NullEngineParams { /// Null engine descriptor #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct NullEngine { /// Ethash params. pub params: NullEngineParams, diff --git a/json/src/spec/params.rs b/json/src/spec/params.rs index ea0a55d49aa..801a5f3a50d 100644 --- a/json/src/spec/params.rs +++ b/json/src/spec/params.rs @@ -22,6 +22,7 @@ use bytes::Bytes; /// Spec params. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Params { /// Account start nonce, defaults to 0. #[serde(rename="accountStartNonce")] diff --git a/json/src/spec/seal.rs b/json/src/spec/seal.rs index b61d141d641..891e00536da 100644 --- a/json/src/spec/seal.rs +++ b/json/src/spec/seal.rs @@ -22,6 +22,7 @@ use bytes::Bytes; /// Ethereum seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Ethereum { /// Seal nonce. pub nonce: H64, @@ -32,6 +33,7 @@ pub struct Ethereum { /// AuthorityRound seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct AuthorityRoundSeal { /// Seal step. pub step: Uint, @@ -41,6 +43,7 @@ pub struct AuthorityRoundSeal { /// Tendermint seal. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct TendermintSeal { /// Seal round. pub round: Uint, @@ -52,6 +55,7 @@ pub struct TendermintSeal { /// Seal variants. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum Seal { /// Ethereum seal. #[serde(rename="ethereum")] diff --git a/json/src/spec/spec.rs b/json/src/spec/spec.rs index a15d1faf6e6..e0a869891b1 100644 --- a/json/src/spec/spec.rs +++ b/json/src/spec/spec.rs @@ -38,6 +38,7 @@ pub enum ForkSpec { /// Spec deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Spec { /// Spec name. pub name: String, @@ -71,6 +72,71 @@ mod tests { use serde_json; use spec::spec::Spec; + #[test] + fn should_error_on_unknown_fields() { + let s = r#"{ + "name": "Morden", + "dataDir": "morden", + "engine": { + "Ethash": { + "params": { + "minimumDifficulty": "0x020000", + "difficultyBoundDivisor": "0x0800", + "durationLimit": "0x0d", + "homesteadTransition" : "0x", + "daoHardforkTransition": "0xffffffffffffffff", + "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", + "daoHardforkAccounts": [] + } + } + }, + "params": { + "accountStartNonce": "0x0100000", + "maximumExtraDataSize": "0x20", + "minGasLimit": "0x1388", + "networkID" : "0x2", + "forkBlock": "0xffffffffffffffff", + "forkCanonHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "gasLimitBoundDivisor": "0x20", + "unknownField": "0x0" + }, + "genesis": { + "seal": { + "ethereum": { + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x00006d6f7264656e" + } + }, + "difficulty": "0x20000", + "author": "0x0000000000000000000000000000000000000000", + "timestamp": "0x00", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x", + "gasLimit": "0x2fefd8" + }, + "nodes": [ + "enode://b1217cbaa440e35ed471157123fe468e19e8b5ad5bedb4b1fdbcbdab6fb2f5ed3e95dd9c24a22a79fdb2352204cea207df27d92bfd21bfd41545e8b16f637499@104.44.138.37:30303" + ], + "accounts": { + "0000000000000000000000000000000000000001": { "balance": "1", "nonce": "1048576", "builtin": { "name": "ecrecover", "pricing": { "linear": { "base": 3000, "word": 0 } } } }, + "0000000000000000000000000000000000000002": { "balance": "1", "nonce": "1048576", "builtin": { "name": "sha256", "pricing": { "linear": { "base": 60, "word": 12 } } } }, + "0000000000000000000000000000000000000003": { "balance": "1", "nonce": "1048576", "builtin": { "name": "ripemd160", "pricing": { "linear": { "base": 600, "word": 120 } } } }, + "0000000000000000000000000000000000000004": { "balance": "1", "nonce": "1048576", "builtin": { "name": "identity", "pricing": { "linear": { "base": 15, "word": 3 } } } }, + "102e61f5d8f9bc71d0ad4a084df4e65e05ce0e1c": { "balance": "1606938044258990275541962092341162602522202993782792835301376", "nonce": "1048576" } + }, + "hardcodedSync": { + "header": "f901f9a0d405da4e66f1445d455195229624e133f5baafe72b5cf7b3c36c12c8146e98b7a01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347948888f1f195afa192cfee860698584c030f4c9db1a05fb2b4bfdef7b314451cb138a534d225c922fc0e5fbe25e451142732c3e25c25a088d2ec6b9860aae1a2c3b299f72b6a5d70d7f7ba4722c78f2c49ba96273c2158a007c6fdfa8eea7e86b81f5b0fc0f78f90cc19f4aa60d323151e0cac660199e9a1b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008302008003832fefba82524d84568e932a80a0a0349d8c3df71f1a48a9df7d03fd5f14aeee7d91332c009ecaff0a71ead405bd88ab4e252a7e8c2a23", + "totalDifficulty": "0x400000000", + "CHTs": [ + "0x11bbe8db4e347b4e8c937c1c8370e4b5ed33adb3db69cbdb7a38e1e50b1b82fa", + "0xd7f8974fb5ac78d9ac099b9ad5018bedc2ce0a72dad1827a1709da30580f0544" + ] + } + }"#; + let result: Result = serde_json::from_str(s); + assert!(result.is_err()); + } + #[test] fn spec_deserialization() { let s = r#"{ @@ -91,7 +157,6 @@ mod tests { }, "params": { "accountStartNonce": "0x0100000", - "homesteadTransition": "0x789b0", "maximumExtraDataSize": "0x20", "minGasLimit": "0x1388", "networkID" : "0x2", diff --git a/json/src/spec/state.rs b/json/src/spec/state.rs index d15ad540ce4..6227750e1f7 100644 --- a/json/src/spec/state.rs +++ b/json/src/spec/state.rs @@ -23,6 +23,7 @@ use spec::{Account, Builtin}; /// Blockchain test state deserializer. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct State(BTreeMap); impl State { diff --git a/json/src/spec/tendermint.rs b/json/src/spec/tendermint.rs index e0a6568aa9c..da437153179 100644 --- a/json/src/spec/tendermint.rs +++ b/json/src/spec/tendermint.rs @@ -21,6 +21,7 @@ use super::ValidatorSet; /// Tendermint params deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct TendermintParams { /// Valid validators. pub validators: ValidatorSet, @@ -43,6 +44,7 @@ pub struct TendermintParams { /// Tendermint engine deserialization. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Tendermint { /// Ethash params. pub params: TendermintParams, diff --git a/json/src/spec/validator_set.rs b/json/src/spec/validator_set.rs index 41fa60961a7..c754f79a951 100644 --- a/json/src/spec/validator_set.rs +++ b/json/src/spec/validator_set.rs @@ -22,6 +22,7 @@ use hash::Address; /// Different ways of specifying validators. #[derive(Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub enum ValidatorSet { /// A simple list of authorities. #[serde(rename="list")] diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 39235c69098..58f8418125b 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -243,6 +243,7 @@ const TRANSACTION_COUNT_SPEC: &'static [u8] = br#"{ "params": { "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", + "blockReward": "0x4563918244F40000", "durationLimit": "0x0d", "homesteadTransition": "0xffffffffffffffff", "daoHardforkTransition": "0xffffffffffffffff", @@ -253,7 +254,6 @@ const TRANSACTION_COUNT_SPEC: &'static [u8] = br#"{ }, "params": { "gasLimitBoundDivisor": "0x0400", - "blockReward": "0x4563918244F40000", "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "accountStartNonce": "0x00", "maximumExtraDataSize": "0x20", From 29f260ceb8662f2ff8553bf18805a25fcfb3e6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Wed, 28 Nov 2018 10:30:05 +0000 Subject: [PATCH 12/18] Improve block and transaction propagation (#9954) * Refactor sync to add priority tasks. * Send priority tasks notifications. * Propagate blocks, optimize transactions. * Implement transaction propagation. Use sync_channel. * Tone down info. * Prevent deadlock by not waiting forever for sync lock. * Fix lock order. * Don't use sync_channel to prevent deadlocks. * Fix tests. --- ethcore/service/src/service.rs | 8 +- ethcore/src/client/chain_notify.rs | 12 +- ethcore/src/client/client.rs | 36 ++- ethcore/src/client/traits.rs | 5 + ethcore/src/miner/miner.rs | 2 +- ethcore/src/verification/queue/mod.rs | 7 + ethcore/sync/src/api.rs | 95 +++++-- ethcore/sync/src/chain/handler.rs | 16 +- ethcore/sync/src/chain/mod.rs | 361 ++++++++++++++++++------- ethcore/sync/src/chain/propagator.rs | 265 +++++++++--------- ethcore/sync/src/chain/supplier.rs | 42 ++- ethcore/sync/src/sync_io.rs | 2 +- ethcore/sync/src/tests/helpers.rs | 4 +- ethcore/sync/src/transactions_stats.rs | 3 +- miner/src/pool/listener.rs | 4 + parity/modules.rs | 34 ++- parity/run.rs | 19 +- util/fastmap/src/lib.rs | 6 +- 18 files changed, 626 insertions(+), 295 deletions(-) diff --git a/ethcore/service/src/service.rs b/ethcore/service/src/service.rs index 1763b8fd5ed..77429a4e56e 100644 --- a/ethcore/service/src/service.rs +++ b/ethcore/service/src/service.rs @@ -106,7 +106,13 @@ impl ClientService { info!("Configured for {} using {} engine", Colour::White.bold().paint(spec.name.clone()), Colour::Yellow.bold().paint(spec.engine.name())); let pruning = config.pruning; - let client = Client::new(config, &spec, blockchain_db.clone(), miner.clone(), io_service.channel())?; + let client = Client::new( + config, + &spec, + blockchain_db.clone(), + miner.clone(), + io_service.channel(), + )?; miner.set_io_channel(io_service.channel()); miner.set_in_chain_checker(&client.clone()); diff --git a/ethcore/src/client/chain_notify.rs b/ethcore/src/client/chain_notify.rs index ebfe7bdef5c..3d576ae12aa 100644 --- a/ethcore/src/client/chain_notify.rs +++ b/ethcore/src/client/chain_notify.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use bytes::Bytes; -use ethereum_types::H256; +use ethereum_types::{H256, U256}; use transaction::UnverifiedTransaction; use blockchain::ImportRoute; use std::time::Duration; @@ -141,7 +141,15 @@ pub trait ChainNotify : Send + Sync { } /// fires when chain broadcasts a message - fn broadcast(&self, _message_type: ChainMessageType) {} + fn broadcast(&self, _message_type: ChainMessageType) { + // does nothing by default + } + + /// fires when new block is about to be imported + /// implementations should be light + fn block_pre_import(&self, _bytes: &Bytes, _hash: &H256, _difficulty: &U256) { + // does nothing by default + } /// fires when new transactions are received from a peer fn transactions_received(&self, diff --git a/ethcore/src/client/client.rs b/ethcore/src/client/client.rs index d5ae9d80f6a..00dc9cab53e 100644 --- a/ethcore/src/client/client.rs +++ b/ethcore/src/client/client.rs @@ -881,7 +881,7 @@ impl Client { /// Flush the block import queue. pub fn flush_queue(&self) { self.importer.block_queue.flush(); - while !self.importer.block_queue.queue_info().is_empty() { + while !self.importer.block_queue.is_empty() { self.import_verified_blocks(); } } @@ -1423,8 +1423,21 @@ impl ImportBlock for Client { bail!(EthcoreErrorKind::Block(BlockError::UnknownParent(unverified.parent_hash()))); } + let raw = if self.importer.block_queue.is_empty() { + Some(( + unverified.bytes.clone(), + unverified.header.hash(), + *unverified.header.difficulty(), + )) + } else { None }; + match self.importer.block_queue.import(unverified) { - Ok(res) => Ok(res), + Ok(hash) => { + if let Some((raw, hash, difficulty)) = raw { + self.notify(move |n| n.block_pre_import(&raw, &hash, &difficulty)); + } + Ok(hash) + }, // we only care about block errors (not import errors) Err((block, EthcoreError(EthcoreErrorKind::Block(err), _))) => { self.importer.bad_blocks.report(block.bytes, format!("{:?}", err)); @@ -1878,6 +1891,10 @@ impl BlockChainClient for Client { self.importer.block_queue.queue_info() } + fn is_queue_empty(&self) -> bool { + self.importer.block_queue.is_empty() + } + fn clear_queue(&self) { self.importer.block_queue.clear(); } @@ -2288,7 +2305,11 @@ impl ScheduleInfo for Client { impl ImportSealedBlock for Client { fn import_sealed_block(&self, block: SealedBlock) -> EthcoreResult { let start = Instant::now(); + let raw = block.rlp_bytes(); let header = block.header().clone(); + let hash = header.hash(); + self.notify(|n| n.block_pre_import(&raw, &hash, header.difficulty())); + let route = { // Do a super duper basic verification to detect potential bugs if let Err(e) = self.engine.verify_block_basic(&header) { @@ -2306,15 +2327,14 @@ impl ImportSealedBlock for Client { let block_data = block.rlp_bytes(); let route = self.importer.commit_block(block, &header, encoded::Block::new(block_data), self); - trace!(target: "client", "Imported sealed block #{} ({})", header.number(), header.hash()); + trace!(target: "client", "Imported sealed block #{} ({})", header.number(), hash); self.state_db.write().sync_cache(&route.enacted, &route.retracted, false); route }; - let h = header.hash(); let route = ChainRoute::from([route].as_ref()); self.importer.miner.chain_new_blocks( self, - &[h], + &[hash], &[], route.enacted(), route.retracted(), @@ -2322,16 +2342,16 @@ impl ImportSealedBlock for Client { ); self.notify(|notify| { notify.new_blocks( - vec![h], + vec![hash], vec![], route.clone(), - vec![h], + vec![hash], vec![], start.elapsed(), ); }); self.db.read().key_value().flush().expect("DB flush failed."); - Ok(h) + Ok(hash) } } diff --git a/ethcore/src/client/traits.rs b/ethcore/src/client/traits.rs index 5b78a54b34e..55d527013ec 100644 --- a/ethcore/src/client/traits.rs +++ b/ethcore/src/client/traits.rs @@ -300,6 +300,11 @@ pub trait BlockChainClient : Sync + Send + AccountData + BlockChain + CallContra /// Get block queue information. fn queue_info(&self) -> BlockQueueInfo; + /// Returns true if block queue is empty. + fn is_queue_empty(&self) -> bool { + self.queue_info().is_empty() + } + /// Clear block queue and abort all import activity. fn clear_queue(&self); diff --git a/ethcore/src/miner/miner.rs b/ethcore/src/miner/miner.rs index 39dac1f2b21..c73887800c8 100644 --- a/ethcore/src/miner/miner.rs +++ b/ethcore/src/miner/miner.rs @@ -576,7 +576,7 @@ impl Miner { trace!(target: "miner", "requires_reseal: sealing enabled"); // Disable sealing if there were no requests for SEALING_TIMEOUT_IN_BLOCKS - let had_requests = sealing.last_request.map(|last_request| + let had_requests = sealing.last_request.map(|last_request| best_block.saturating_sub(last_request) <= SEALING_TIMEOUT_IN_BLOCKS ).unwrap_or(false); diff --git a/ethcore/src/verification/queue/mod.rs b/ethcore/src/verification/queue/mod.rs index 9b1597439b0..b9242f47b06 100644 --- a/ethcore/src/verification/queue/mod.rs +++ b/ethcore/src/verification/queue/mod.rs @@ -583,6 +583,13 @@ impl VerificationQueue { result } + /// Returns true if there is nothing currently in the queue. + /// TODO [ToDr] Optimize to avoid locking + pub fn is_empty(&self) -> bool { + let v = &self.verification; + v.unverified.lock().is_empty() && v.verifying.lock().is_empty() && v.verified.lock().is_empty() + } + /// Get queue status. pub fn queue_info(&self) -> QueueInfo { use std::mem::size_of; diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 4296a4c2d07..6fef887a0fe 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc, atomic}; use std::collections::{HashMap, BTreeMap}; use std::io; use std::ops::Range; @@ -33,10 +33,10 @@ use ethcore::client::{BlockChainClient, ChainNotify, ChainRoute, ChainMessageTyp use ethcore::snapshot::SnapshotService; use ethcore::header::BlockNumber; use sync_io::NetSyncIo; -use chain::{ChainSync, SyncStatus as EthSyncStatus}; +use chain::{ChainSyncApi, SyncStatus as EthSyncStatus}; use std::net::{SocketAddr, AddrParseError}; use std::str::FromStr; -use parking_lot::RwLock; +use parking_lot::{RwLock, Mutex}; use chain::{ETH_PROTOCOL_VERSION_63, ETH_PROTOCOL_VERSION_62, PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; @@ -228,6 +228,37 @@ impl AttachedProtocol { } } +/// A prioritized tasks run in a specialised timer. +/// Every task should be completed within a hard deadline, +/// if it's not it's either cancelled or split into multiple tasks. +/// NOTE These tasks might not complete at all, so anything +/// that happens here should work even if the task is cancelled. +#[derive(Debug)] +pub enum PriorityTask { + /// Propagate given block + PropagateBlock { + /// When the task was initiated + started: ::std::time::Instant, + /// Raw block RLP to propagate + block: Bytes, + /// Block hash + hash: H256, + /// Blocks difficulty + difficulty: U256, + }, + /// Propagate a list of transactions + PropagateTransactions(::std::time::Instant, Arc), +} +impl PriorityTask { + /// Mark the task as being processed, right after it's retrieved from the queue. + pub fn starting(&self) { + match *self { + PriorityTask::PropagateTransactions(_, ref is_ready) => is_ready.store(true, atomic::Ordering::SeqCst), + _ => {}, + } + } +} + /// EthSync initialization parameters. pub struct Params { /// Configuration. @@ -260,6 +291,8 @@ pub struct EthSync { subprotocol_name: [u8; 3], /// Light subprotocol name. light_subprotocol_name: [u8; 3], + /// Priority tasks notification channel + priority_tasks: Mutex>, } fn light_params( @@ -312,13 +345,19 @@ impl EthSync { }) }; - let chain_sync = ChainSync::new(params.config, &*params.chain, params.private_tx_handler.clone()); + let (priority_tasks_tx, priority_tasks_rx) = mpsc::channel(); + let sync = ChainSyncApi::new( + params.config, + &*params.chain, + params.private_tx_handler.clone(), + priority_tasks_rx, + ); let service = NetworkService::new(params.network_config.clone().into_basic()?, connection_filter)?; let sync = Arc::new(EthSync { network: service, eth_handler: Arc::new(SyncProtocolHandler { - sync: RwLock::new(chain_sync), + sync, chain: params.chain, snapshot_service: params.snapshot_service, overlay: RwLock::new(HashMap::new()), @@ -327,26 +366,32 @@ impl EthSync { subprotocol_name: params.config.subprotocol_name, light_subprotocol_name: params.config.light_subprotocol_name, attached_protos: params.attached_protos, + priority_tasks: Mutex::new(priority_tasks_tx), }); Ok(sync) } + + /// Priority tasks producer + pub fn priority_tasks(&self) -> mpsc::Sender { + self.priority_tasks.lock().clone() + } } impl SyncProvider for EthSync { /// Get sync status fn status(&self) -> EthSyncStatus { - self.eth_handler.sync.read().status() + self.eth_handler.sync.status() } /// Get sync peers fn peers(&self) -> Vec { self.network.with_context_eval(self.subprotocol_name, |ctx| { let peer_ids = self.network.connected_peers(); - let eth_sync = self.eth_handler.sync.read(); let light_proto = self.light_proto.as_ref(); - peer_ids.into_iter().filter_map(|peer_id| { + let peer_info = self.eth_handler.sync.peer_info(&peer_ids); + peer_ids.into_iter().zip(peer_info).filter_map(|(peer_id, peer_info)| { let session_info = match ctx.session_info(peer_id) { None => return None, Some(info) => info, @@ -358,7 +403,7 @@ impl SyncProvider for EthSync { capabilities: session_info.peer_capabilities.into_iter().map(|c| c.to_string()).collect(), remote_address: session_info.remote_address, local_address: session_info.local_address, - eth_info: eth_sync.peer_info(&peer_id), + eth_info: peer_info, pip_info: light_proto.as_ref().and_then(|lp| lp.peer_status(peer_id)).map(Into::into), }) }).collect() @@ -370,17 +415,16 @@ impl SyncProvider for EthSync { } fn transactions_stats(&self) -> BTreeMap { - let sync = self.eth_handler.sync.read(); - sync.transactions_stats() - .iter() - .map(|(hash, stats)| (*hash, stats.into())) - .collect() + self.eth_handler.sync.transactions_stats() } } const PEERS_TIMER: TimerToken = 0; const SYNC_TIMER: TimerToken = 1; const TX_TIMER: TimerToken = 2; +const PRIORITY_TIMER: TimerToken = 3; + +pub(crate) const PRIORITY_TIMER_INTERVAL: Duration = Duration::from_millis(250); struct SyncProtocolHandler { /// Shared blockchain client. @@ -388,7 +432,7 @@ struct SyncProtocolHandler { /// Shared snapshot service. snapshot_service: Arc, /// Sync strategy - sync: RwLock, + sync: ChainSyncApi, /// Chain overlay used to cache data such as fork block. overlay: RwLock>, } @@ -399,11 +443,13 @@ impl NetworkProtocolHandler for SyncProtocolHandler { io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer"); io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer"); + + io.register_timer(PRIORITY_TIMER, PRIORITY_TIMER_INTERVAL).expect("Error registering peers timer"); } } fn read(&self, io: &NetworkContext, peer: &PeerId, packet_id: u8, data: &[u8]) { - ChainSync::dispatch_packet(&self.sync, &mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); + self.sync.dispatch_packet(&mut NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay), *peer, packet_id, data); } fn connected(&self, io: &NetworkContext, peer: &PeerId) { @@ -429,15 +475,26 @@ impl NetworkProtocolHandler for SyncProtocolHandler { match timer { PEERS_TIMER => self.sync.write().maintain_peers(&mut io), SYNC_TIMER => self.sync.write().maintain_sync(&mut io), - TX_TIMER => { - self.sync.write().propagate_new_transactions(&mut io); - }, + TX_TIMER => self.sync.write().propagate_new_transactions(&mut io), + PRIORITY_TIMER => self.sync.process_priority_queue(&mut io), _ => warn!("Unknown timer {} triggered.", timer), } } } impl ChainNotify for EthSync { + fn block_pre_import(&self, bytes: &Bytes, hash: &H256, difficulty: &U256) { + let task = PriorityTask::PropagateBlock { + started: ::std::time::Instant::now(), + block: bytes.clone(), + hash: *hash, + difficulty: *difficulty, + }; + if let Err(e) = self.priority_tasks.lock().send(task) { + warn!(target: "sync", "Unexpected error during priority block propagation: {:?}", e); + } + } + fn new_blocks(&self, imported: Vec, invalid: Vec, diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index e6702974349..104a80320e9 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -29,7 +29,6 @@ use rlp::Rlp; use snapshot::ChunkType; use std::cmp; use std::mem; -use std::collections::HashSet; use std::time::Instant; use sync_io::SyncIo; @@ -58,7 +57,6 @@ use super::{ SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, STATUS_PACKET, - TRANSACTIONS_PACKET, }; /// The Chain Sync Handler: handles responses from peers @@ -67,14 +65,9 @@ pub struct SyncHandler; impl SyncHandler { /// Handle incoming packet from peer pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) { - debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); - return; - } let rlp = Rlp::new(data); let result = match packet_id { STATUS_PACKET => SyncHandler::on_peer_status(sync, io, peer, &rlp), - TRANSACTIONS_PACKET => SyncHandler::on_peer_transactions(sync, io, peer, &rlp), BLOCK_HEADERS_PACKET => SyncHandler::on_peer_block_headers(sync, io, peer, &rlp), BLOCK_BODIES_PACKET => SyncHandler::on_peer_block_bodies(sync, io, peer, &rlp), RECEIPTS_PACKET => SyncHandler::on_peer_block_receipts(sync, io, peer, &rlp), @@ -109,10 +102,9 @@ impl SyncHandler { } /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) { trace!(target: "sync", "Received consensus packet from {:?}", peer_id); io.chain().queue_consensus_message(r.as_raw().to_vec()); - Ok(()) } /// Called by peer when it is disconnecting @@ -578,8 +570,8 @@ impl SyncHandler { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: if sync.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, asking_snapshot_data: None, @@ -635,7 +627,7 @@ impl SyncHandler { } /// Called when peer sends us new transactions - fn on_peer_transactions(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), DownloaderImportError> { + pub fn on_peer_transactions(sync: &ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { // Accept transactions only when fully synced if !io.is_chain_queue_empty() || (sync.state != SyncState::Idle && sync.state != SyncState::NewBlocks) { trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id); diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index e0fc8ecddb6..cdedd56303c 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -92,17 +92,17 @@ mod propagator; mod requester; mod supplier; -use std::sync::Arc; -use std::collections::{HashSet, HashMap}; +use std::sync::{Arc, mpsc}; +use std::collections::{HashSet, HashMap, BTreeMap}; use std::cmp; use std::time::{Duration, Instant}; use hash::keccak; use heapsize::HeapSizeOf; use ethereum_types::{H256, U256}; -use fastmap::H256FastMap; -use parking_lot::RwLock; +use fastmap::{H256FastMap, H256FastSet}; +use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; use bytes::Bytes; -use rlp::{Rlp, RlpStream, DecoderError}; +use rlp::{RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; @@ -112,7 +112,7 @@ use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, DownloadAction}; use rand::Rng; use snapshot::{Snapshot}; -use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID}; +use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID, PriorityTask}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; @@ -120,7 +120,7 @@ use transaction::UnverifiedTransaction; use self::handler::SyncHandler; use self::propagator::SyncPropagator; use self::requester::SyncRequester; -use self::supplier::SyncSupplier; +pub(crate) use self::supplier::SyncSupplier; known_heap_size!(0, PeerInfo); @@ -187,6 +187,11 @@ const FORK_HEADER_TIMEOUT: Duration = Duration::from_secs(3); const SNAPSHOT_MANIFEST_TIMEOUT: Duration = Duration::from_secs(5); const SNAPSHOT_DATA_TIMEOUT: Duration = Duration::from_secs(120); +/// Defines how much time we have to complete priority transaction or block propagation. +/// after the deadline is reached the task is considered finished +/// (so we might sent only to some part of the peers we originally intended to send to) +const PRIORITY_TASK_DEADLINE: Duration = Duration::from_millis(100); + #[derive(Copy, Clone, Eq, PartialEq, Debug)] /// Sync state pub enum SyncState { @@ -323,9 +328,9 @@ pub struct PeerInfo { /// Request timestamp ask_time: Instant, /// Holds a set of transactions recently sent to this peer to avoid spamming. - last_sent_transactions: HashSet, + last_sent_transactions: H256FastSet, /// Holds a set of private transactions and their signatures recently sent to this peer to avoid spamming. - last_sent_private_transactions: HashSet, + last_sent_private_transactions: H256FastSet, /// Pending request is expired and result should be ignored expired: bool, /// Peer fork confirmation status @@ -375,6 +380,217 @@ pub mod random { pub type RlpResponseResult = Result, PacketDecodeError>; pub type Peers = HashMap; +/// Thread-safe wrapper for `ChainSync`. +/// +/// NOTE always lock in order of fields declaration +pub struct ChainSyncApi { + /// Priority tasks queue + priority_tasks: Mutex>, + /// The rest of sync data + sync: RwLock, +} + +impl ChainSyncApi { + /// Creates new `ChainSyncApi` + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + priority_tasks: mpsc::Receiver, + ) -> Self { + ChainSyncApi { + sync: RwLock::new(ChainSync::new(config, chain, private_tx_handler)), + priority_tasks: Mutex::new(priority_tasks), + } + } + + /// Gives `write` access to underlying `ChainSync` + pub fn write(&self) -> RwLockWriteGuard { + self.sync.write() + } + + /// Returns info about given list of peers + pub fn peer_info(&self, ids: &[PeerId]) -> Vec> { + let sync = self.sync.read(); + ids.iter().map(|id| sync.peer_info(id)).collect() + } + + /// Returns synchonization status + pub fn status(&self) -> SyncStatus { + self.sync.read().status() + } + + /// Returns transactions propagation statistics + pub fn transactions_stats(&self) -> BTreeMap { + self.sync.read().transactions_stats() + .iter() + .map(|(hash, stats)| (*hash, stats.into())) + .collect() + } + + /// Dispatch incoming requests and responses + pub fn dispatch_packet(&self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + SyncSupplier::dispatch_packet(&self.sync, io, peer, packet_id, data) + } + + /// Process a priority propagation queue. + /// This task is run from a timer and should be time constrained. + /// Hence we set up a deadline for the execution and cancel the task if the deadline is exceeded. + /// + /// NOTE This method should only handle stuff that can be canceled and would reach other peers + /// by other means. + pub fn process_priority_queue(&self, io: &mut SyncIo) { + fn check_deadline(deadline: Instant) -> Option { + let now = Instant::now(); + if now > deadline { + None + } else { + Some(deadline - now) + } + } + + // deadline to get the task from the queue + let deadline = Instant::now() + ::api::PRIORITY_TIMER_INTERVAL; + let mut work = || { + let task = { + let tasks = self.priority_tasks.try_lock_until(deadline)?; + let left = check_deadline(deadline)?; + tasks.recv_timeout(left).ok()? + }; + task.starting(); + // wait for the sync lock until deadline, + // note we might drop the task here if we won't manage to acquire the lock. + let mut sync = self.sync.try_write_until(deadline)?; + // since we already have everything let's use a different deadline + // to do the rest of the job now, so that previous work is not wasted. + let deadline = Instant::now() + PRIORITY_TASK_DEADLINE; + let as_ms = move |prev| { + let dur: Duration = Instant::now() - prev; + dur.as_secs() * 1_000 + dur.subsec_millis() as u64 + }; + match task { + // NOTE We can't simply use existing methods, + // cause the block is not in the DB yet. + PriorityTask::PropagateBlock { started, block, hash, difficulty } => { + // try to send to peers that are on the same block as us + // (they will most likely accept the new block). + let chain_info = io.chain().chain_info(); + let total_difficulty = chain_info.total_difficulty + difficulty; + let rlp = ChainSync::create_block_rlp(&block, total_difficulty); + for peers in sync.get_peers(&chain_info, PeerState::SameBlock).chunks(10) { + check_deadline(deadline)?; + for peer in peers { + SyncPropagator::send_packet(io, *peer, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer) { + peer.latest_hash = hash; + } + } + } + debug!(target: "sync", "Finished block propagation, took {}ms", as_ms(started)); + }, + PriorityTask::PropagateTransactions(time, _) => { + SyncPropagator::propagate_new_transactions(&mut sync, io, || { + check_deadline(deadline).is_some() + }); + debug!(target: "sync", "Finished transaction propagation, took {}ms", as_ms(time)); + }, + } + + Some(()) + }; + + // Process as many items as we can until the deadline is reached. + loop { + if work().is_none() { + return; + } + } + } +} + +// Static methods +impl ChainSync { + /// creates rlp to send for the tree defined by 'from' and 'to' hashes + fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { + match chain.tree_route(from, to) { + Some(route) => { + let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); + match route.blocks.len() { + 0 => None, + _ => { + let mut blocks = route.blocks; + blocks.extend(uncles); + let mut rlp_stream = RlpStream::new_list(blocks.len()); + for block_hash in blocks { + let mut hash_rlp = RlpStream::new_list(2); + let number = chain.block_header(BlockId::Hash(block_hash.clone())) + .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); + hash_rlp.append(&block_hash); + hash_rlp.append(&number); + rlp_stream.append_raw(hash_rlp.as_raw(), 1); + } + Some(rlp_stream.out()) + } + } + }, + None => None + } + } + + /// creates rlp from block bytes and total difficulty + fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { + let mut rlp_stream = RlpStream::new_list(2); + rlp_stream.append_raw(bytes, 1); + rlp_stream.append(&total_difficulty); + rlp_stream.out() + } + + /// creates latest block rlp for the given client + fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) + .expect("Best block always exists").into_inner(), + chain.chain_info().total_difficulty + ) + } + + /// creates given hash block rlp for the given client + fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { + Self::create_block_rlp( + &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), + chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") + ) + } + + fn select_random_peers(peers: &[PeerId]) -> Vec { + // take sqrt(x) peers + let mut peers = peers.to_vec(); + let mut count = (peers.len() as f64).powf(0.5).round() as usize; + count = cmp::min(count, MAX_PEERS_PROPAGATION); + count = cmp::max(count, MIN_PEERS_PROPAGATION); + random::new().shuffle(&mut peers); + peers.truncate(count); + peers + } + + fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { + let best_block = chain.chain_info().best_block_number; + match warp_sync { + WarpSync::Enabled => SyncState::WaitingPeers, + WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, + _ => SyncState::Idle, + } + } +} + +/// A peer query method for getting a list of peers +enum PeerState { + /// Peer is on different hash than us + Lagging, + /// Peer is on the same block as us + SameBlock +} + /// Blockchain sync handler. /// See module documentation for more details. pub struct ChainSync { @@ -417,10 +633,14 @@ pub struct ChainSync { impl ChainSync { /// Create a new instance of syncing strategy. - pub fn new(config: SyncConfig, chain: &BlockChainClient, private_tx_handler: Arc) -> ChainSync { + pub fn new( + config: SyncConfig, + chain: &BlockChainClient, + private_tx_handler: Arc, + ) -> Self { let chain_info = chain.chain_info(); let best_block = chain.chain_info().best_block_number; - let state = ChainSync::get_init_state(config.warp_sync, chain); + let state = Self::get_init_state(config.warp_sync, chain); let mut sync = ChainSync { state, @@ -445,15 +665,6 @@ impl ChainSync { sync } - fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { - let best_block = chain.chain_info().best_block_number; - match warp_sync { - WarpSync::Enabled => SyncState::WaitingPeers, - WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, - _ => SyncState::Idle, - } - } - /// Returns synchonization status pub fn status(&self) -> SyncStatus { let last_imported_number = self.new_blocks.last_imported_block_number(); @@ -521,7 +732,7 @@ impl ChainSync { } } } - self.state = state.unwrap_or_else(|| ChainSync::get_init_state(self.warp_sync, io.chain())); + self.state = state.unwrap_or_else(|| Self::get_init_state(self.warp_sync, io.chain())); // Reactivate peers only if some progress has been made // since the last sync round of if starting fresh. self.active_peers = self.peers.keys().cloned().collect(); @@ -1004,67 +1215,24 @@ impl ChainSync { } } - /// creates rlp to send for the tree defined by 'from' and 'to' hashes - fn create_new_hashes_rlp(chain: &BlockChainClient, from: &H256, to: &H256) -> Option { - match chain.tree_route(from, to) { - Some(route) => { - let uncles = chain.find_uncles(from).unwrap_or_else(Vec::new); - match route.blocks.len() { - 0 => None, - _ => { - let mut blocks = route.blocks; - blocks.extend(uncles); - let mut rlp_stream = RlpStream::new_list(blocks.len()); - for block_hash in blocks { - let mut hash_rlp = RlpStream::new_list(2); - let number = chain.block_header(BlockId::Hash(block_hash.clone())) - .expect("chain.tree_route and chain.find_uncles only return hahses of blocks that are in the blockchain. qed.").number(); - hash_rlp.append(&block_hash); - hash_rlp.append(&number); - rlp_stream.append_raw(hash_rlp.as_raw(), 1); - } - Some(rlp_stream.out()) - } - } - }, - None => None - } - } - - /// creates rlp from block bytes and total difficulty - fn create_block_rlp(bytes: &Bytes, total_difficulty: U256) -> Bytes { - let mut rlp_stream = RlpStream::new_list(2); - rlp_stream.append_raw(bytes, 1); - rlp_stream.append(&total_difficulty); - rlp_stream.out() - } - - /// creates latest block rlp for the given client - fn create_latest_block_rlp(chain: &BlockChainClient) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(chain.chain_info().best_block_hash)) - .expect("Best block always exists").into_inner(), - chain.chain_info().total_difficulty - ) + /// returns peer ids that have different block than our chain + fn get_lagging_peers(&self, chain_info: &BlockChainInfo) -> Vec { + self.get_peers(chain_info, PeerState::Lagging) } - /// creates given hash block rlp for the given client - fn create_new_block_rlp(chain: &BlockChainClient, hash: &H256) -> Bytes { - ChainSync::create_block_rlp( - &chain.block(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed").into_inner(), - chain.block_total_difficulty(BlockId::Hash(hash.clone())).expect("Block has just been sealed; qed.") - ) - } - - /// returns peer ids that have different blocks than our chain - fn get_lagging_peers(&mut self, chain_info: &BlockChainInfo) -> Vec { + /// returns peer ids that have different or the same blocks than our chain + fn get_peers(&self, chain_info: &BlockChainInfo, peers: PeerState) -> Vec { let latest_hash = chain_info.best_block_hash; self .peers - .iter_mut() + .iter() .filter_map(|(&id, ref mut peer_info)| { trace!(target: "sync", "Checking peer our best {} their best {}", latest_hash, peer_info.latest_hash); - if peer_info.latest_hash != latest_hash { + let matches = match peers { + PeerState::Lagging => peer_info.latest_hash != latest_hash, + PeerState::SameBlock => peer_info.latest_hash == latest_hash, + }; + if matches { Some(id) } else { None @@ -1073,17 +1241,6 @@ impl ChainSync { .collect::>() } - fn select_random_peers(peers: &[PeerId]) -> Vec { - // take sqrt(x) peers - let mut peers = peers.to_vec(); - let mut count = (peers.len() as f64).powf(0.5).round() as usize; - count = cmp::min(count, MAX_PEERS_PROPAGATION); - count = cmp::max(count, MIN_PEERS_PROPAGATION); - random::new().shuffle(&mut peers); - peers.truncate(count); - peers - } - fn get_consensus_peers(&self) -> Vec { self.peers.iter().filter_map(|(id, p)| if p.protocol_version >= PAR_PROTOCOL_VERSION_2.0 { Some(*id) } else { None }).collect() } @@ -1132,21 +1289,10 @@ impl ChainSync { } } - /// Dispatch incoming requests and responses - pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) - } - pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); SyncHandler::on_packet(self, io, peer, packet_id, data); } - /// Called when peer sends us new consensus packet - pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - SyncHandler::on_consensus_packet(io, peer_id, r) - } - /// Called by peer when it is disconnecting pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { SyncHandler::on_peer_aborting(self, io, peer); @@ -1158,8 +1304,16 @@ impl ChainSync { } /// propagates new transactions to all peers - pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize { - SyncPropagator::propagate_new_transactions(self, io) + pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) { + let deadline = Instant::now() + Duration::from_millis(500); + SyncPropagator::propagate_new_transactions(self, io, || { + if deadline > Instant::now() { + true + } else { + debug!(target: "sync", "Wasn't able to finish transaction propagation within a deadline."); + false + } + }); } /// Broadcast consensus message to peers. @@ -1175,7 +1329,7 @@ impl ChainSync { #[cfg(test)] pub mod tests { - use std::collections::{HashSet, VecDeque}; + use std::collections::{VecDeque}; use ethkey; use network::PeerId; use tests::helpers::{TestIo}; @@ -1291,8 +1445,8 @@ pub mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: super::ForkConfirmation::Confirmed, snapshot_number: None, @@ -1307,7 +1461,7 @@ pub mod tests { fn finds_lagging_peers() { let mut client = TestBlockChainClient::new(); client.add_blocks(100, EachBlockWith::Uncle); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); + let sync = dummy_sync_with_peer(client.block_hash_delta_minus(10), &client); let chain_info = client.chain_info(); let lagging_peers = sync.get_lagging_peers(&chain_info); @@ -1447,3 +1601,4 @@ pub mod tests { assert_eq!(status.status.transaction_count, 0); } } + diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 82b592c4ba0..689ccfc02b1 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -18,6 +18,7 @@ use bytes::Bytes; use ethereum_types::H256; use ethcore::client::BlockChainInfo; use ethcore::header::BlockNumber; +use fastmap::H256FastSet; use network::{PeerId, PacketId}; use rand::Rng; use rlp::{Encodable, RlpStream}; @@ -69,49 +70,51 @@ impl SyncPropagator { /// propagates latest block to a set of peers pub fn propagate_blocks(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewBlocks to {:?}", peers); - let mut sent = 0; - for peer_id in peers { - if blocks.is_empty() { - let rlp = ChainSync::create_latest_block_rlp(io.chain()); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); - } else { - for h in blocks { - let rlp = ChainSync::create_new_block_rlp(io.chain(), h); - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); + let sent = peers.len(); + let mut send_packet = |io: &mut SyncIo, rlp: Bytes| { + for peer_id in peers { + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone()); + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = chain_info.best_block_hash.clone(); } } - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); + }; + + if blocks.is_empty() { + let rlp = ChainSync::create_latest_block_rlp(io.chain()); + send_packet(io, rlp); + } else { + for h in blocks { + let rlp = ChainSync::create_new_block_rlp(io.chain(), h); + send_packet(io, rlp); } - sent += 1; } + sent } /// propagates new known hashes to all peers pub fn propagate_new_hashes(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize { trace!(target: "sync", "Sending NewHashes to {:?}", peers); - let mut sent = 0; let last_parent = *io.chain().best_block_header().parent_hash(); + let best_block_hash = chain_info.best_block_hash; + let rlp = match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &best_block_hash) { + Some(rlp) => rlp, + None => return 0 + }; + + let sent = peers.len(); for peer_id in peers { - sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) { - Some(rlp) => { - { - if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); - } - } - SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp); - 1 - }, - None => 0 + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = best_block_hash; } + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp.clone()); } sent } /// propagates new transactions to all peers - pub fn propagate_new_transactions(sync: &mut ChainSync, io: &mut SyncIo) -> usize { + pub fn propagate_new_transactions bool>(sync: &mut ChainSync, io: &mut SyncIo, mut should_continue: F) -> usize { // Early out if nobody to send to. if sync.peers.is_empty() { return 0; @@ -122,6 +125,10 @@ impl SyncPropagator { return 0; } + if !should_continue() { + return 0; + } + let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter() .map(|tx| tx.signed()) .partition(|tx| !tx.gas_price.is_zero()); @@ -130,24 +137,34 @@ impl SyncPropagator { let mut affected_peers = HashSet::new(); if !transactions.is_empty() { let peers = SyncPropagator::select_peers_for_transactions(sync, |_| true); - affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, peers, transactions); + affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, peers, transactions, &mut should_continue, + ); } // most of times service_transactions will be empty // => there's no need to merge packets if !service_transactions.is_empty() { let service_transactions_peers = SyncPropagator::select_peers_for_transactions(sync, |peer_id| accepts_service_transaction(&io.peer_info(*peer_id))); - let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, service_transactions_peers, service_transactions); + let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers( + sync, io, service_transactions_peers, service_transactions, &mut should_continue + ); affected_peers.extend(&service_transactions_affected_peers); } affected_peers.len() } - fn propagate_transactions_to_peers(sync: &mut ChainSync, io: &mut SyncIo, peers: Vec, transactions: Vec<&SignedTransaction>) -> HashSet { + fn propagate_transactions_to_peers bool>( + sync: &mut ChainSync, + io: &mut SyncIo, + peers: Vec, + transactions: Vec<&SignedTransaction>, + mut should_continue: F, + ) -> HashSet { let all_transactions_hashes = transactions.iter() .map(|tx| tx.hash()) - .collect::>(); + .collect::(); let all_transactions_rlp = { let mut packet = RlpStream::new_list(transactions.len()); for tx in &transactions { packet.append(&**tx); } @@ -157,102 +174,104 @@ impl SyncPropagator { // Clear old transactions from stats sync.transactions_stats.retain(&all_transactions_hashes); - // sqrt(x)/x scaled to max u32 - let block_number = io.chain().chain_info().best_block_number; + let send_packet = |io: &mut SyncIo, peer_id: PeerId, sent: usize, rlp: Bytes| { + let size = rlp.len(); + SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); + trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); + }; - let lucky_peers = { - peers.into_iter() - .filter_map(|peer_id| { - let stats = &mut sync.transactions_stats; - let peer_info = sync.peers.get_mut(&peer_id) - .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - - // Send all transactions - if peer_info.last_sent_transactions.is_empty() { - // update stats - for hash in &all_transactions_hashes { - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - stats.propagated(hash, id, block_number); - } - peer_info.last_sent_transactions = all_transactions_hashes.clone(); - return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone())); - } + let block_number = io.chain().chain_info().best_block_number; + let mut sent_to_peers = HashSet::new(); + let mut max_sent = 0; - // Get hashes of all transactions to send to this peer - let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) - .cloned() - .collect::>(); - if to_send.is_empty() { - return None; - } + // for every peer construct and send transactions packet + for peer_id in peers { + if !should_continue() { + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + return sent_to_peers; + } - // Construct RLP - let (packet, to_send) = { - let mut to_send = to_send; - let mut packet = RlpStream::new(); - packet.begin_unbounded_list(); - let mut pushed = 0; - for tx in &transactions { - let hash = tx.hash(); - if to_send.contains(&hash) { - let mut transaction = RlpStream::new(); - tx.rlp_append(&mut transaction); - let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); - if !appended { - // Maximal packet size reached just proceed with sending - debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); - to_send = to_send.into_iter().take(pushed).collect(); - break; - } - pushed += 1; - } - } - packet.complete_unbounded_list(); - (packet, to_send) - }; + let stats = &mut sync.transactions_stats; + let peer_info = sync.peers.get_mut(&peer_id) + .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - // Update stats + // Send all transactions, if the peer doesn't know about anything + if peer_info.last_sent_transactions.is_empty() { + // update stats + for hash in &all_transactions_hashes { let id = io.peer_session_info(peer_id).and_then(|info| info.id); - for hash in &to_send { - // update stats - stats.propagated(hash, id, block_number); - } + stats.propagated(hash, id, block_number); + } + peer_info.last_sent_transactions = all_transactions_hashes.clone(); - peer_info.last_sent_transactions = all_transactions_hashes - .intersection(&peer_info.last_sent_transactions) - .chain(&to_send) - .cloned() - .collect(); - Some((peer_id, to_send.len(), packet.out())) - }) - .collect::>() - }; + send_packet(io, peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, all_transactions_hashes.len()); + continue; + } + + // Get hashes of all transactions to send to this peer + let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) + .cloned() + .collect::>(); + if to_send.is_empty() { + continue; + } + + // Construct RLP + let (packet, to_send) = { + let mut to_send = to_send; + let mut packet = RlpStream::new(); + packet.begin_unbounded_list(); + let mut pushed = 0; + for tx in &transactions { + let hash = tx.hash(); + if to_send.contains(&hash) { + let mut transaction = RlpStream::new(); + tx.rlp_append(&mut transaction); + let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); + if !appended { + // Maximal packet size reached just proceed with sending + debug!(target: "sync", "Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); + to_send = to_send.into_iter().take(pushed).collect(); + break; + } + pushed += 1; + } + } + packet.complete_unbounded_list(); + (packet, to_send) + }; - // Send RLPs - let mut peers = HashSet::new(); - if lucky_peers.len() > 0 { - let mut max_sent = 0; - let lucky_peers_len = lucky_peers.len(); - for (peer_id, sent, rlp) in lucky_peers { - peers.insert(peer_id); - let size = rlp.len(); - SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); - trace!(target: "sync", "{:02} <- Transactions ({} entries; {} bytes)", peer_id, sent, size); - max_sent = cmp::max(max_sent, sent); + // Update stats + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + for hash in &to_send { + // update stats + stats.propagated(hash, id, block_number); } - debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len); + + peer_info.last_sent_transactions = all_transactions_hashes + .intersection(&peer_info.last_sent_transactions) + .chain(&to_send) + .cloned() + .collect(); + send_packet(io, peer_id, to_send.len(), packet.out()); + sent_to_peers.insert(peer_id); + max_sent = cmp::max(max_sent, to_send.len()); + } - peers + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, sent_to_peers.len()); + sent_to_peers } pub fn propagate_latest_blocks(sync: &mut ChainSync, io: &mut SyncIo, sealed: &[H256]) { let chain_info = io.chain().chain_info(); if (((chain_info.best_block_number as i64) - (sync.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let mut peers = sync.get_lagging_peers(&chain_info); + let peers = sync.get_lagging_peers(&chain_info); if sealed.is_empty() { let hashes = SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers); - peers = ChainSync::select_random_peers(&peers); + let peers = ChainSync::select_random_peers(&peers); let blocks = SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers); if blocks != 0 || hashes != 0 { trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); @@ -318,7 +337,7 @@ impl SyncPropagator { } /// Generic packet sender - fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { + pub fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { if let Err(e) = sync.send(peer_id, packet_id, packet) { debug!(target:"sync", "Error sending packet: {:?}", e); sync.disconnect_peer(peer_id); @@ -419,8 +438,8 @@ mod tests { asking_blocks: Vec::new(), asking_hash: None, ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - last_sent_private_transactions: HashSet::new(), + last_sent_transactions: Default::default(), + last_sent_private_transactions: Default::default(), expired: false, confirmation: ForkConfirmation::Confirmed, snapshot_number: None, @@ -447,13 +466,13 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // Even after new block transactions should not be propagated twice sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the third time - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -474,7 +493,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); io.chain.insert_transaction_to_queue(); // New block import should not trigger propagation. // (we only propagate on timeout) @@ -498,10 +517,10 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(0, io.packets.len()); assert_eq!(0, peer_count); @@ -519,7 +538,7 @@ mod tests { // should sent some { let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); assert_eq!(1, io.packets.len()); assert_eq!(1, peer_count); } @@ -528,9 +547,9 @@ mod tests { let (peer_count2, peer_count3) = { let mut io = TestIo::new(&mut client, &ss, &queue, None); // Propagate new transactions - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // And now the peer should have all transactions - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); (peer_count2, peer_count3) }; @@ -553,7 +572,7 @@ mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); let stats = sync.transactions_stats(); assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") @@ -583,7 +602,7 @@ mod tests { io.peers_info.insert(4, "Parity-Ethereum/v2.7.3-ABCDEFGH".to_owned()); // and new service transaction is propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // peer#2 && peer#4 are receiving service transaction assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET @@ -607,7 +626,7 @@ mod tests { io.peers_info.insert(1, "Parity-Ethereum/v2.6".to_owned()); // and service + non-service transactions are propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io, || true); // two separate packets for peer are queued: // 1) with non-service-transaction diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 4bce0ef9850..eaee584cad0 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -27,6 +27,7 @@ use sync_io::SyncIo; use super::{ ChainSync, + SyncHandler, RlpResponseResult, PacketDecodeError, BLOCK_BODIES_PACKET, @@ -47,6 +48,8 @@ use super::{ RECEIPTS_PACKET, SNAPSHOT_DATA_PACKET, SNAPSHOT_MANIFEST_PACKET, + STATUS_PACKET, + TRANSACTIONS_PACKET, }; /// The Chain Sync Supplier: answers requests from peers with available data @@ -56,6 +59,7 @@ impl SyncSupplier { /// Dispatch incoming requests and responses pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { let rlp = Rlp::new(data); + let result = match packet_id { GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_block_bodies, @@ -80,9 +84,39 @@ impl SyncSupplier { GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_snapshot_data, |e| format!("Error sending snapshot data: {:?}", e)), - CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), - _ => { + + STATUS_PACKET => { sync.write().on_packet(io, peer, packet_id, data); + Ok(()) + }, + // Packets that require the peer to be confirmed + _ => { + if !sync.read().peers.contains_key(&peer) { + debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); + return; + } + debug!(target: "sync", "{} -> Dispatching packet: {}", peer, packet_id); + + match packet_id { + CONSENSUS_DATA_PACKET => { + SyncHandler::on_consensus_packet(io, peer, &rlp) + }, + TRANSACTIONS_PACKET => { + let res = { + let sync_ro = sync.read(); + SyncHandler::on_peer_transactions(&*sync_ro, io, peer, &rlp) + }; + if res.is_err() { + // peer sent invalid data, disconnect. + io.disable_peer(peer); + sync.write().deactivate_peer(io, peer); + } + }, + _ => { + sync.write().on_packet(io, peer, packet_id, data); + } + } + Ok(()) } }; @@ -404,7 +438,7 @@ mod test { io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); assert_eq!(1, io.packets.len()); } @@ -446,7 +480,7 @@ mod test { assert_eq!(603, rlp_result.unwrap().1.out().len()); io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); + SyncSupplier::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); assert_eq!(1, io.packets.len()); } } diff --git a/ethcore/sync/src/sync_io.rs b/ethcore/sync/src/sync_io.rs index c7704724c66..a5e9f7b2f44 100644 --- a/ethcore/sync/src/sync_io.rs +++ b/ethcore/sync/src/sync_io.rs @@ -52,7 +52,7 @@ pub trait SyncIo { fn protocol_version(&self, protocol: &ProtocolId, peer_id: PeerId) -> u8; /// Returns if the chain block queue empty fn is_chain_queue_empty(&self) -> bool { - self.chain().queue_info().is_empty() + self.chain().is_queue_empty() } /// Check if the session is expired fn is_expired(&self) -> bool; diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index d75d71ea90a..d83597e5239 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -33,7 +33,7 @@ use ethcore::test_helpers; use sync_io::SyncIo; use io::{IoChannel, IoContext, IoHandler}; use api::WARP_SYNC_PROTOCOL_ID; -use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET}; +use chain::{ChainSync, ETH_PROTOCOL_VERSION_63, PAR_PROTOCOL_VERSION_3, PRIVATE_TRANSACTION_PACKET, SIGNED_PRIVATE_TRANSACTION_PACKET, SyncSupplier}; use SyncConfig; use private_tx::SimplePrivateTxHandler; @@ -271,7 +271,7 @@ impl Peer for EthPeer { fn receive_message(&self, from: PeerId, msg: TestPacket) -> HashSet { let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, Some(from)); - ChainSync::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); + SyncSupplier::dispatch_packet(&self.sync, &mut io, from, msg.packet_id, &msg.data); self.chain.flush(); io.to_disconnect.clone() } diff --git a/ethcore/sync/src/transactions_stats.rs b/ethcore/sync/src/transactions_stats.rs index 4d11dcf6809..7d5e2ca4a86 100644 --- a/ethcore/sync/src/transactions_stats.rs +++ b/ethcore/sync/src/transactions_stats.rs @@ -15,6 +15,7 @@ // along with Parity. If not, see . use api::TransactionStats; +use std::hash::BuildHasher; use std::collections::{HashSet, HashMap}; use ethereum_types::{H256, H512}; use fastmap::H256FastMap; @@ -74,7 +75,7 @@ impl TransactionsStats { } /// Retains only transactions present in given `HashSet`. - pub fn retain(&mut self, hashes: &HashSet) { + pub fn retain(&mut self, hashes: &HashSet) { let to_remove = self.pending_transactions.keys() .filter(|hash| !hashes.contains(hash)) .cloned() diff --git a/miner/src/pool/listener.rs b/miner/src/pool/listener.rs index 5776ba84532..f5e76e15d52 100644 --- a/miner/src/pool/listener.rs +++ b/miner/src/pool/listener.rs @@ -50,6 +50,10 @@ impl Notifier { /// Notify listeners about all currently pending transactions. pub fn notify(&mut self) { + if self.pending.is_empty() { + return; + } + for l in &self.listeners { (l)(&self.pending); } diff --git a/parity/modules.rs b/parity/modules.rs index e12e8ee4583..ac84aea5f21 100644 --- a/parity/modules.rs +++ b/parity/modules.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use std::sync::Arc; +use std::sync::{Arc, mpsc}; use ethcore::client::BlockChainClient; use sync::{self, AttachedProtocol, SyncConfig, NetworkConfiguration, Params, ConnectionFilter}; @@ -25,12 +25,17 @@ pub use sync::{EthSync, SyncProvider, ManageNetwork, PrivateTxHandler}; pub use ethcore::client::ChainNotify; use ethcore_logger::Config as LogConfig; -pub type SyncModules = (Arc, Arc, Arc); +pub type SyncModules = ( + Arc, + Arc, + Arc, + mpsc::Sender, +); pub fn sync( - sync_cfg: SyncConfig, - net_cfg: NetworkConfiguration, - client: Arc, + config: SyncConfig, + network_config: NetworkConfiguration, + chain: Arc, snapshot_service: Arc, private_tx_handler: Arc, provider: Arc, @@ -39,15 +44,20 @@ pub fn sync( connection_filter: Option>, ) -> Result { let eth_sync = EthSync::new(Params { - config: sync_cfg, - chain: client, - provider: provider, - snapshot_service: snapshot_service, + config, + chain, + provider, + snapshot_service, private_tx_handler, - network_config: net_cfg, - attached_protos: attached_protos, + network_config, + attached_protos, }, connection_filter)?; - Ok((eth_sync.clone() as Arc, eth_sync.clone() as Arc, eth_sync.clone() as Arc)) + Ok(( + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.clone() as Arc, + eth_sync.priority_tasks() + )) } diff --git a/parity/run.rs b/parity/run.rs index 7ee0250d6bd..14fd58dd107 100644 --- a/parity/run.rs +++ b/parity/run.rs @@ -15,7 +15,7 @@ // along with Parity. If not, see . use std::any::Any; -use std::sync::{Arc, Weak}; +use std::sync::{Arc, Weak, atomic}; use std::time::{Duration, Instant}; use std::thread; @@ -479,7 +479,6 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: cmd.gas_pricer_conf.to_gas_pricer(fetch.clone(), runtime.executor()), &spec, Some(account_provider.clone()), - )); miner.set_author(cmd.miner_extras.author, None).expect("Fails only if password is Some; password is None; qed"); miner.set_gas_range_target(cmd.miner_extras.gas_range_target); @@ -636,7 +635,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: }; // create sync object - let (sync_provider, manage_network, chain_notify) = modules::sync( + let (sync_provider, manage_network, chain_notify, priority_tasks) = modules::sync( sync_config, net_conf.clone().into(), client.clone(), @@ -650,6 +649,18 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: service.add_notify(chain_notify.clone()); + // Propagate transactions as soon as they are imported. + let tx = ::parking_lot::Mutex::new(priority_tasks); + let is_ready = Arc::new(atomic::AtomicBool::new(true)); + miner.add_transactions_listener(Box::new(move |_hashes| { + // we want to have only one PendingTransactions task in the queue. + if is_ready.compare_and_swap(true, false, atomic::Ordering::SeqCst) { + let task = ::sync::PriorityTask::PropagateTransactions(Instant::now(), is_ready.clone()); + // we ignore error cause it means that we are closing + let _ = tx.lock().send(task); + } + })); + // provider not added to a notification center is effectively disabled // TODO [debris] refactor it later on if cmd.private_tx_enabled { @@ -735,7 +746,7 @@ fn execute_impl(cmd: RunCmd, logger: Arc, on_client_rq: let secretstore_deps = secretstore::Dependencies { client: client.clone(), sync: sync_provider.clone(), - miner: miner, + miner: miner.clone(), account_provider: account_provider, accounts_passwords: &passwords, }; diff --git a/util/fastmap/src/lib.rs b/util/fastmap/src/lib.rs index 135ce54babe..65dd9dfb4a0 100644 --- a/util/fastmap/src/lib.rs +++ b/util/fastmap/src/lib.rs @@ -21,11 +21,13 @@ extern crate plain_hasher; use ethereum_types::H256; use std::hash; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use plain_hasher::PlainHasher; /// Specialized version of `HashMap` with H256 keys and fast hashing function. pub type H256FastMap = HashMap>; +/// Specialized version of HashSet with H256 values and fast hashing function. +pub type H256FastSet = HashSet>; #[cfg(test)] mod tests { @@ -36,4 +38,4 @@ mod tests { let mut h = H256FastMap::default(); h.insert(H256::from(123), "abc"); } -} \ No newline at end of file +} From 4d1c67eacd276af3784c68a1db4cdac5ab25c827 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Wed, 28 Nov 2018 12:19:35 +0100 Subject: [PATCH 13/18] Fix unstable peers and slowness in sync (#9967) * Don't sync all peers after each response * Update formating * Fix tests: add `continue_sync` to `Sync_step` * Update ethcore/sync/src/chain/mod.rs Co-Authored-By: ngotchac --- ethcore/sync/src/api.rs | 13 +++++---- ethcore/sync/src/chain/handler.rs | 2 -- ethcore/sync/src/chain/mod.rs | 46 +++++++++++++++---------------- ethcore/sync/src/tests/helpers.rs | 8 ++++-- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/ethcore/sync/src/api.rs b/ethcore/sync/src/api.rs index 6fef887a0fe..7474d79b37f 100644 --- a/ethcore/sync/src/api.rs +++ b/ethcore/sync/src/api.rs @@ -420,9 +420,10 @@ impl SyncProvider for EthSync { } const PEERS_TIMER: TimerToken = 0; -const SYNC_TIMER: TimerToken = 1; -const TX_TIMER: TimerToken = 2; -const PRIORITY_TIMER: TimerToken = 3; +const MAINTAIN_SYNC_TIMER: TimerToken = 1; +const CONTINUE_SYNC_TIMER: TimerToken = 2; +const TX_TIMER: TimerToken = 3; +const PRIORITY_TIMER: TimerToken = 4; pub(crate) const PRIORITY_TIMER_INTERVAL: Duration = Duration::from_millis(250); @@ -441,7 +442,8 @@ impl NetworkProtocolHandler for SyncProtocolHandler { fn initialize(&self, io: &NetworkContext) { if io.subprotocol_name() != WARP_SYNC_PROTOCOL_ID { io.register_timer(PEERS_TIMER, Duration::from_millis(700)).expect("Error registering peers timer"); - io.register_timer(SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(MAINTAIN_SYNC_TIMER, Duration::from_millis(1100)).expect("Error registering sync timer"); + io.register_timer(CONTINUE_SYNC_TIMER, Duration::from_millis(2500)).expect("Error registering sync timer"); io.register_timer(TX_TIMER, Duration::from_millis(1300)).expect("Error registering transactions timer"); io.register_timer(PRIORITY_TIMER, PRIORITY_TIMER_INTERVAL).expect("Error registering peers timer"); @@ -474,7 +476,8 @@ impl NetworkProtocolHandler for SyncProtocolHandler { let mut io = NetSyncIo::new(io, &*self.chain, &*self.snapshot_service, &self.overlay); match timer { PEERS_TIMER => self.sync.write().maintain_peers(&mut io), - SYNC_TIMER => self.sync.write().maintain_sync(&mut io), + MAINTAIN_SYNC_TIMER => self.sync.write().maintain_sync(&mut io), + CONTINUE_SYNC_TIMER => self.sync.write().continue_sync(&mut io), TX_TIMER => self.sync.write().propagate_new_transactions(&mut io), PRIORITY_TIMER => self.sync.process_priority_queue(&mut io), _ => warn!("Unknown timer {} triggered.", timer), diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index 104a80320e9..e1518d4f522 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -97,8 +97,6 @@ impl SyncHandler { sync.sync_peer(io, peer, false); }, } - // give tasks to other peers - sync.continue_sync(io); } /// Called when peer sends us new consensus packet diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index cdedd56303c..a01b25528f0 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -866,37 +866,35 @@ impl ChainSync { } /// Resume downloading - fn continue_sync(&mut self, io: &mut SyncIo) { - // Collect active peers that can sync - let confirmed_peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| - if peer.can_sync() { - Some((*peer_id, peer.protocol_version)) - } else { - None - } - ).collect(); - - trace!( - target: "sync", - "Syncing with peers: {} active, {} confirmed, {} total", - self.active_peers.len(), confirmed_peers.len(), self.peers.len() - ); - + pub fn continue_sync(&mut self, io: &mut SyncIo) { if self.state == SyncState::Waiting { trace!(target: "sync", "Waiting for the block queue"); } else if self.state == SyncState::SnapshotWaiting { trace!(target: "sync", "Waiting for the snapshot restoration"); } else { - let mut peers: Vec<(PeerId, u8)> = confirmed_peers.iter().filter(|&&(peer_id, _)| - self.active_peers.contains(&peer_id) - ).map(|v| *v).collect(); + // Collect active peers that can sync + let mut peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| + if peer.can_sync() && peer.asking == PeerAsking::Nothing && self.active_peers.contains(&peer_id) { + Some((*peer_id, peer.protocol_version)) + } else { + None + } + ).collect(); - random::new().shuffle(&mut peers); //TODO: sort by rating - // prefer peers with higher protocol version - peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); + if peers.len() > 0 { + trace!( + target: "sync", + "Syncing with peers: {} active, {} available, {} total", + self.active_peers.len(), peers.len(), self.peers.len() + ); - for (peer_id, _) in peers { - self.sync_peer(io, peer_id, false); + random::new().shuffle(&mut peers); // TODO (#646): sort by rating + // prefer peers with higher protocol version + peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); + + for (peer_id, _) in peers { + self.sync_peer(io, peer_id, false); + } } } diff --git a/ethcore/sync/src/tests/helpers.rs b/ethcore/sync/src/tests/helpers.rs index d83597e5239..3eac91a0db1 100644 --- a/ethcore/sync/src/tests/helpers.rs +++ b/ethcore/sync/src/tests/helpers.rs @@ -286,10 +286,12 @@ impl Peer for EthPeer { } fn sync_step(&self) { + let mut io = TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None); self.chain.flush(); - self.sync.write().maintain_peers(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().maintain_sync(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); - self.sync.write().propagate_new_transactions(&mut TestIo::new(&*self.chain, &self.snapshot_service, &self.queue, None)); + self.sync.write().maintain_peers(&mut io); + self.sync.write().maintain_sync(&mut io); + self.sync.write().continue_sync(&mut io); + self.sync.write().propagate_new_transactions(&mut io); } fn restart_sync(&self) { From f62288430a5f98b98dbf0340309020d51060c518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= Date: Wed, 28 Nov 2018 12:41:09 +0000 Subject: [PATCH 14/18] fix rpc middlewares --- rpc/src/v1/extractors.rs | 3 ++- rpc/src/v1/informant.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rpc/src/v1/extractors.rs b/rpc/src/v1/extractors.rs index f0433db0297..a7c1c40b1ea 100644 --- a/rpc/src/v1/extractors.rs +++ b/rpc/src/v1/extractors.rs @@ -218,9 +218,10 @@ impl> WsDispatcher { impl> core::Middleware for WsDispatcher { type Future = Either< - core::FutureRpcResult, + core::FutureRpcResult, core::FutureResponse, >; + type CallFuture = core::middleware::NoopCallFuture; fn on_request(&self, request: core::Request, meta: Metadata, process: F) -> Either diff --git a/rpc/src/v1/informant.rs b/rpc/src/v1/informant.rs index 3dab548437f..dfa20ee4695 100644 --- a/rpc/src/v1/informant.rs +++ b/rpc/src/v1/informant.rs @@ -205,6 +205,7 @@ impl Middleware { impl core::Middleware for Middleware { type Future = core::FutureResponse; + type CallFuture = core::middleware::NoopCallFuture; fn on_request(&self, request: core::Request, meta: M, process: F) -> Either where F: FnOnce(core::Request, M) -> X, From 9c9bfafa4c13d82a6de172b2090fd19beacd4335 Mon Sep 17 00:00:00 2001 From: Andronik Ordian Date: Wed, 28 Nov 2018 15:44:26 +0300 Subject: [PATCH 15/18] fix Cargo.lock --- Cargo.lock | 217 ++++++----------------------------------------------- 1 file changed, 22 insertions(+), 195 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4a7c03391a7..e5e6c31b9d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -185,11 +185,6 @@ dependencies = [ "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "cast" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "cc" version = "1.0.25" @@ -289,51 +284,6 @@ dependencies = [ "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "criterion" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", - "criterion-plot 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "criterion-stats 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", - "csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", - "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "handlebars 0.32.4 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools-num 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", - "simplelog 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "walkdir 2.2.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "criterion-plot" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "criterion-stats" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", - "thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "crossbeam" version = "0.3.2" @@ -407,23 +357,6 @@ name = "crunchy" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "csv" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "csv-core" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ct-logs" version = "0.4.0" @@ -475,14 +408,14 @@ dependencies = [ [[package]] name = "docopt" -version = "1.0.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -497,6 +430,7 @@ dependencies = [ "ethabi 6.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -600,7 +534,6 @@ dependencies = [ name = "ethash" version = "1.12.0" dependencies = [ - "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -633,7 +566,6 @@ dependencies = [ "bn 0.4.4 (git+https://github.com/paritytech/bn)", "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", "common-types 0.1.0", - "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -684,8 +616,6 @@ dependencies = [ "rlp_compress 0.1.0", "rlp_derive 0.1.0", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", "stats 0.1.0", "stop-guard 0.1.0", "tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -774,7 +704,7 @@ dependencies = [ "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -919,7 +849,6 @@ dependencies = [ "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-runtime 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-hex 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1074,7 +1003,7 @@ dependencies = [ name = "ethkey-cli" version = "0.1.0" dependencies = [ - "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethkey 0.3.0", "panic_hook 0.1.0", @@ -1115,7 +1044,7 @@ name = "ethstore-cli" version = "0.1.0" dependencies = [ "dir 0.1.2", - "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethstore 0.2.0", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", @@ -1131,7 +1060,6 @@ name = "evm" version = "0.1.0" dependencies = [ "bit-set 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1148,7 +1076,7 @@ dependencies = [ name = "evmbin" version = "0.1.0" dependencies = [ - "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.12.0", "ethcore-transaction 0.1.0", @@ -1361,21 +1289,6 @@ name = "hamming" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "handlebars" -version = "0.32.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)", - "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 1.0.80 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "hardware-wallet" version = "1.12.0" @@ -1603,14 +1516,6 @@ dependencies = [ "either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "itertools-num" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "itoa" version = "0.4.3" @@ -2257,7 +2162,7 @@ version = "1.12.0" dependencies = [ "jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "panic_hook 0.1.0", - "parity-ethereum 2.3.0", + "parity-ethereum 2.2.2", ] [[package]] @@ -2273,7 +2178,7 @@ dependencies = [ [[package]] name = "parity-ethereum" -version = "2.3.0" +version = "2.2.2" dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2282,7 +2187,7 @@ dependencies = [ "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)", "daemonize 0.2.3 (git+https://github.com/paritytech/daemonize)", "dir 0.1.2", - "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore 1.12.0", "ethcore-io 1.12.0", @@ -2322,11 +2227,11 @@ dependencies = [ "parity-rpc-client 1.4.0", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.3.0", + "parity-version 2.2.2", "parity-whisper 0.1.0", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "registrar 0.0.1", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "rpassword 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2477,7 +2382,7 @@ dependencies = [ "parity-crypto 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-runtime 0.1.0", "parity-updater 1.12.0", - "parity-version 2.3.0", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2574,7 +2479,7 @@ dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-hash-fetch 1.12.0", "parity-path 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "parity-version 2.3.0", + "parity-version 2.2.2", "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2584,7 +2489,7 @@ dependencies = [ [[package]] name = "parity-version" -version = "2.3.0" +version = "2.2.2" dependencies = [ "parity-bytes 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rlp 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2696,21 +2601,6 @@ name = "percent-encoding" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "pest" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "pest_derive" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)", - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "petgraph" version = "0.4.13" @@ -2878,11 +2768,6 @@ name = "quick-error" version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "quote" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "quote" version = "0.6.8" @@ -3272,16 +3157,6 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "simplelog" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "term 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "siphasher" version = "0.1.3" @@ -3358,18 +3233,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "strsim" -version = "0.7.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "syn" -version = "0.11.11" +name = "strsim" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] [[package]] name = "syn" @@ -3381,14 +3251,6 @@ dependencies = [ "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "synom" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "synstructure" version = "0.10.1" @@ -3414,15 +3276,6 @@ dependencies = [ "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "term" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "term_size" version = "0.3.1" @@ -3477,11 +3330,6 @@ dependencies = [ "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "thread-scoped" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "thread_local" version = "0.3.6" @@ -3896,11 +3744,6 @@ name = "unicode-width" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "unicode-xid" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "unicode-xid" version = "0.1.0" @@ -4084,7 +3927,7 @@ dependencies = [ name = "whisper-cli" version = "0.1.0" dependencies = [ - "docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-logger 1.12.0", "ethcore-network 1.12.0", "ethcore-network-devp2p 1.12.0", @@ -4215,7 +4058,6 @@ dependencies = [ "checksum byte-tools 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" "checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" "checksum bytes 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "0ce55bd354b095246fc34caf4e9e242f5297a7fd938b090cadfea6eee614aa62" -"checksum cast 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "926013f2860c46252efceabb19f4a6b308197505082c609025aa6706c011d427" "checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" "checksum cesu8 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" "checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" @@ -4225,9 +4067,6 @@ dependencies = [ "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" "checksum cmake 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)" = "6ec65ee4f9c9d16f335091d23693457ed4928657ba4982289d7fafee03bc614a" "checksum combine 3.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fc1d011beeed29187b8db2ac3925c8dd4d3e87db463dc9d2d2833985388fc5bc" -"checksum criterion 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c47d2b548c5647e1a436dc0cb78d4ebf51b6bf7ab101ed76662828bdd4d3a24a" -"checksum criterion-plot 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6e649d6aacdbbdb94ec659561a309a71336fc5655ed408f3afd28df2fc0c4f4f" -"checksum criterion-stats 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ff43cac80562f91ead0b617c1be74edf350adfaa195809d355de98dfc8f9237d" "checksum crossbeam 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "24ce9782d4d5c53674646a6a4c1863a21a8fc0cb649b3c94dfc16e45071dea19" "checksum crossbeam-deque 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f739f8c5363aca78cfb059edf753d8f0d36908c348f3d8d1503f03d8b75d9cf3" "checksum crossbeam-deque 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3486aefc4c0487b9cb52372c97df0a48b8c249514af1ee99703bf70d2f2ceda1" @@ -4237,14 +4076,12 @@ dependencies = [ "checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015" "checksum crunchy 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a2f4a431c5c9f662e1200b7c7f02c34e91361150e382089a8f2dec3ba680cbda" "checksum crunchy 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c240f247c278fa08a6d4820a6a222bfc6e0d999e51ba67be94f44c905b2161f2" -"checksum csv 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d54f6b0fd69128a2894b1a3e57af5849a0963c1cc77b165d30b896e40296452" -"checksum csv-core 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4dd8e6d86f7ba48b4276ef1317edc8cc36167546d8972feb4a2b5fec0b374105" "checksum ct-logs 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "95a4bf5107667e12bf6ce31a3a5066d67acc88942b6742117a41198734aaccaa" "checksum ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)" = "" "checksum daemonize 0.2.3 (git+https://github.com/paritytech/daemonize)" = "" "checksum difference 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3304d19798a8e067e48d8e69b2c37f0b5e9b4e462504ad9e27e9f3fce02bba8" "checksum digest 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" -"checksum docopt 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "db2906c2579b5b7207fc1e328796a9a8835dc44e22dbe8e460b1d636f9a7b225" +"checksum docopt 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d8acd393692c503b168471874953a2531df0e9ab77d0b6bbc582395743300a4a" "checksum edit-distance 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3bd26878c3d921f89797a4e1a1711919f999a9f6946bb6f5a4ffda126d297b7e" "checksum either 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3be565ca5c557d7f59e7cfcf1844f9e3033650c929c6566f511e8005f205c1d0" "checksum elastic-array 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "88d4851b005ef16de812ea9acdb7bece2f0a40dd86c07b85631d7dafa54537bb" @@ -4277,7 +4114,6 @@ dependencies = [ "checksum globset 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4743617a7464bbda3c8aec8558ff2f9429047e025771037df561d383337ff865" "checksum h2 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "a27e7ed946e8335bdf9a191bc1b9b14a03ba822d013d2f58437f4fabcbd7fc2c" "checksum hamming 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" -"checksum handlebars 0.32.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d89ec99d1594f285d4590fc32bac5f75cdab383f1123d504d27862c644a807dd" "checksum hashdb 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d91261ee336dd046ac7df28306cb297b7a7228bd1ae25e9a57f4ed5e0ab628c7" "checksum heapsize 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1679e6ea370dee694f91f1dc469bf94cf8f52051d147aec3e1f9497c6fc22461" "checksum heck 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea04fa3ead4e05e51a7c806fc07271fdbde4e246a6c6d1efd52e72230b771b82" @@ -4300,7 +4136,6 @@ dependencies = [ "checksum ipnetwork 0.12.8 (registry+https://github.com/rust-lang/crates.io-index)" = "70783119ac90828aaba91eae39db32c6c1b8838deea3637e5238efa0130801ab" "checksum itertools 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4833d6978da405305126af4ac88569b5d71ff758581ce5a987dbfa3755f694fc" "checksum itertools 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)" = "f58856976b776fedd95533137617a02fb25719f40e7d9b01c7043cd65474f450" -"checksum itertools-num 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "83ca7b70b838f2e34bc6c2f367a1ed1cfe34fb82464adecadd31cdcc7da882fc" "checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" "checksum jni 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1ecfa3b81afc64d9a6539c4eece96ac9a93c551c713a313800dade8e33d7b5c1" "checksum jni-sys 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" @@ -4377,8 +4212,6 @@ dependencies = [ "checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c" "checksum patricia-trie 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "10438ba40c2f6e9ceca55277d8e7f6a5dafd58cabd802e6d97e16f02aab83a03" "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" -"checksum pest 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" -"checksum pest_derive 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)" = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4" "checksum petgraph 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)" = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" "checksum phf 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "cec29da322b242f4c3098852c77a0ca261c9c01b806cae85a5572a1eb94db9a6" "checksum phf_codegen 0.7.23 (registry+https://github.com/rust-lang/crates.io-index)" = "7d187f00cd98d5afbcd8898f6cf181743a449162aeb329dcd2f3849009e605ad" @@ -4396,7 +4229,6 @@ dependencies = [ "checksum pulldown-cmark 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8361e81576d2e02643b04950e487ec172b687180da65c731c03cf336784e6c07" "checksum pwasm-utils 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "90d2b3c5bf24275fc77db6b14ec00a7a085d8ff9d1c4215fb6f6263e8d7b01bc" "checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" -"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" "checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" "checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1" "checksum rand 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8356f47b32624fef5b3301c1be97e5944ecdd595409cc5da11d05f211db6cfbd" @@ -4440,7 +4272,6 @@ dependencies = [ "checksum sha1 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "171698ce4ec7cbb93babeb3190021b4d72e96ccb98e33d277ae4ea959d6f2d9e" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" "checksum shell32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9ee04b46101f57121c9da2b151988283b6beb79b34f5bb29a58ee48cb695122c" -"checksum simplelog 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e95345f185d5adeb8ec93459d2dc99654e294cc6ccf5b75414d8ea262de9a13" "checksum siphasher 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "833011ca526bd88f16778d32c699d325a9ad302fa06381cd66f7be63351d3f6d" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum skeptic 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "24ebf8a06f5f8bae61ae5bbc7af7aac4ef6907ae975130faba1199e5fe82256a" @@ -4451,21 +4282,18 @@ dependencies = [ "checksum socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c4d11a52082057d87cb5caa31ad812f4504b97ab44732cd8359df2e9ff9f48e7" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" "checksum string 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00caf261d6f90f588f8450b8e1230fa0d5be49ee6140fdfbcb55335aff350970" +"checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550" -"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum syn 0.15.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b036b7b35e846707c0e55c2c9441fa47867c0f87fca416921db3261b1d8c741a" -"checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum target_info 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -"checksum term 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5e6b677dd1e8214ea1ef4297f85dbcbed8e8cdddb561040cc998ca2551c37561" "checksum term_size 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9e5b9a66db815dcfd2da92db471106457082577c3c278d4138ab3e3b4e189327" "checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" "checksum thread-id 3.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c7fbf4c9d56b320106cd64fd024dadfa0be7cb4706725fc44a7d7ce952d820c1" -"checksum thread-scoped 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bcbb6aa301e5d3b0b5ef639c9a9c7e2f1c944f177b460c04dc24c69b1fa2bd99" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" "checksum threadpool 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e2f0c90a5f3459330ac8bc0d2f879c693bb7a2f59689c1083fc4ef83834da865" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" @@ -4508,7 +4336,6 @@ dependencies = [ "checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" "checksum unicode-segmentation 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "aa6024fc12ddfd1c6dbc14a80fa2324d4568849869b779f6bd37e5e4c03344d1" "checksum unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "882386231c45df4700b275c7ff55b6f3698780a650026380e72dabe76fa46526" -"checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" "checksum unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" "checksum untrusted 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "55cd1f4b4e96b46aeb8d4855db4a7a9bd96eeeb5c6a1ab54593328761642ce2f" From 7366b033d971a71ae76286bac176662645857dee Mon Sep 17 00:00:00 2001 From: 5chdn <5chdn@users.noreply.github.com> Date: Wed, 28 Nov 2018 13:52:00 +0100 Subject: [PATCH 16/18] json: resolve merge in spec --- json/src/spec/engine.rs | 1 - json/src/spec/genesis.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/json/src/spec/engine.rs b/json/src/spec/engine.rs index 4dec7324cb5..2026fd25c8f 100644 --- a/json/src/spec/engine.rs +++ b/json/src/spec/engine.rs @@ -90,7 +90,6 @@ mod tests { "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", - "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "homesteadTransition" : "0x", "daoHardforkTransition": "0xffffffffffffffff", "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", diff --git a/json/src/spec/genesis.rs b/json/src/spec/genesis.rs index 5ec86eb26ee..cae2098d32e 100644 --- a/json/src/spec/genesis.rs +++ b/json/src/spec/genesis.rs @@ -71,7 +71,6 @@ mod tests { #[test] fn genesis_deserialization() { let s = r#"{ - "nonce": "0x0000000000000042", "difficulty": "0x400000000", "seal": { "ethereum": { From dc97f848d35a66af4fcec2439d80698d92dc197d Mon Sep 17 00:00:00 2001 From: 5chdn <5chdn@users.noreply.github.com> Date: Wed, 28 Nov 2018 14:28:36 +0100 Subject: [PATCH 17/18] rpc: fix starting_nonce_test --- rpc/src/v1/tests/eth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/src/v1/tests/eth.rs b/rpc/src/v1/tests/eth.rs index 58f8418125b..1a8f3248a2d 100644 --- a/rpc/src/v1/tests/eth.rs +++ b/rpc/src/v1/tests/eth.rs @@ -292,6 +292,7 @@ const POSITIVE_NONCE_SPEC: &'static [u8] = br#"{ "minimumDifficulty": "0x020000", "difficultyBoundDivisor": "0x0800", "durationLimit": "0x0d", + "blockReward": "0x4563918244F40000", "homesteadTransition": "0xffffffffffffffff", "daoHardforkTransition": "0xffffffffffffffff", "daoHardforkBeneficiary": "0x0000000000000000000000000000000000000000", @@ -301,7 +302,6 @@ const POSITIVE_NONCE_SPEC: &'static [u8] = br#"{ }, "params": { "gasLimitBoundDivisor": "0x0400", - "blockReward": "0x4563918244F40000", "registrar" : "0xc6d9d2cd449a754c494264e1809c50e34d64562b", "accountStartNonce": "0x0100", "maximumExtraDataSize": "0x20", From bd45a9c785aa2282c205992e288a0144b0c03594 Mon Sep 17 00:00:00 2001 From: 5chdn <5chdn@users.noreply.github.com> Date: Wed, 28 Nov 2018 15:12:03 +0100 Subject: [PATCH 18/18] ci: allow nightl job to fail --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb712e70e59..e2cb77606d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -152,3 +152,4 @@ test-nightly: - scripts/gitlab/test-all.sh nightly tags: - rust-nightly + allow_failure: true