From 1f735cfa6210c3fac5e723f31d68a61ed7a17b7a Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 11:03:28 +0200 Subject: [PATCH 01/16] Start dividing sync chain : first supplier method --- ethcore/sync/src/{chain.rs => chain/mod.rs} | 151 +--------------- ethcore/sync/src/chain/supplier.rs | 186 ++++++++++++++++++++ 2 files changed, 194 insertions(+), 143 deletions(-) rename ethcore/sync/src/{chain.rs => chain/mod.rs} (94%) create mode 100644 ethcore/sync/src/chain/supplier.rs diff --git a/ethcore/sync/src/chain.rs b/ethcore/sync/src/chain/mod.rs similarity index 94% rename from ethcore/sync/src/chain.rs rename to ethcore/sync/src/chain/mod.rs index eaab13b6f4b..b96574621e2 100644 --- a/ethcore/sync/src/chain.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -88,6 +88,8 @@ /// All other messages are ignored. /// +mod supplier; + use std::sync::Arc; use std::collections::{HashSet, HashMap}; use std::cmp; @@ -115,6 +117,8 @@ use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; +use self::supplier::SyncSupplier; + known_heap_size!(0, PeerInfo); type PacketDecodeError = DecoderError; @@ -131,7 +135,7 @@ pub const PAR_PROTOCOL_VERSION_2: u8 = 2; pub const PAR_PROTOCOL_VERSION_3: u8 = 3; const MAX_BODIES_TO_SEND: usize = 256; -const MAX_HEADERS_TO_SEND: usize = 512; +pub const MAX_HEADERS_TO_SEND: usize = 512; const MAX_NODE_DATA_TO_SEND: usize = 1024; const MAX_RECEIPTS_TO_SEND: usize = 1024; const MAX_RECEIPTS_HEADERS_TO_SEND: usize = 256; @@ -153,7 +157,7 @@ const STATUS_PACKET: u8 = 0x00; const NEW_BLOCK_HASHES_PACKET: u8 = 0x01; const TRANSACTIONS_PACKET: u8 = 0x02; const GET_BLOCK_HEADERS_PACKET: u8 = 0x03; -const BLOCK_HEADERS_PACKET: u8 = 0x04; +pub const BLOCK_HEADERS_PACKET: u8 = 0x04; const GET_BLOCK_BODIES_PACKET: u8 = 0x05; const BLOCK_BODIES_PACKET: u8 = 0x06; const NEW_BLOCK_PACKET: u8 = 0x07; @@ -402,7 +406,7 @@ pub struct ChainSync { warp_sync: WarpSync, } -type RlpResponseResult = Result, PacketDecodeError>; +pub type RlpResponseResult = Result, PacketDecodeError>; impl ChainSync { /// Create a new instance of syncing strategy. @@ -1554,79 +1558,6 @@ impl ChainSync { io.respond(STATUS_PACKET, packet.out()) } - /// Respond to GetBlockHeaders request - fn return_block_headers(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - // Packet layout: - // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] - let max_headers: usize = r.val_at(1)?; - let skip: usize = r.val_at(2)?; - let reverse: bool = r.val_at(3)?; - let last = io.chain().chain_info().best_block_number; - let number = if r.at(0)?.size() == 32 { - // id is a hash - let hash: H256 = r.val_at(0)?; - trace!(target: "sync", "{} -> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", peer_id, hash, max_headers, skip, reverse); - match io.chain().block_header(BlockId::Hash(hash)) { - Some(hdr) => { - let number = hdr.number().into(); - debug_assert_eq!(hdr.hash(), hash); - - if max_headers == 1 || io.chain().block_hash(BlockId::Number(number)) != Some(hash) { - // Non canonical header or single header requested - // TODO: handle single-step reverse hashchains of non-canon hashes - trace!(target:"sync", "Returning single header: {:?}", hash); - let mut rlp = RlpStream::new_list(1); - rlp.append_raw(&hdr.into_inner(), 1); - return Ok(Some((BLOCK_HEADERS_PACKET, rlp))); - } - number - } - None => return Ok(Some((BLOCK_HEADERS_PACKET, RlpStream::new_list(0)))) //no such header, return nothing - } - } else { - trace!(target: "sync", "{} -> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", peer_id, r.val_at::(0)?, max_headers, skip, reverse); - r.val_at(0)? - }; - - let mut number = if reverse { - cmp::min(last, number) - } else { - cmp::max(0, number) - }; - let max_count = cmp::min(MAX_HEADERS_TO_SEND, max_headers); - let mut count = 0; - let mut data = Bytes::new(); - let inc = (skip + 1) as BlockNumber; - let overlay = io.chain_overlay().read(); - - while number <= last && count < max_count { - if let Some(hdr) = overlay.get(&number) { - trace!(target: "sync", "{}: Returning cached fork header", peer_id); - data.extend_from_slice(hdr); - count += 1; - } else if let Some(hdr) = io.chain().block_header(BlockId::Number(number)) { - data.append(&mut hdr.into_inner()); - count += 1; - } else { - // No required block. - break; - } - if reverse { - if number <= inc || number == 0 { - break; - } - number -= inc; - } - else { - number += inc; - } - } - let mut rlp = RlpStream::new_list(count as usize); - rlp.append_raw(&data, count as usize); - trace!(target: "sync", "{} -> GetBlockHeaders: returned {} entries", peer_id, count); - Ok(Some((BLOCK_HEADERS_PACKET, rlp))) - } - /// Respond to GetBlockBodies request fn return_block_bodies(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { let mut count = r.item_count().unwrap_or(0); @@ -1764,7 +1695,7 @@ impl ChainSync { |e| format!("Error sending block bodies: {:?}", e)), GET_BLOCK_HEADERS_PACKET => ChainSync::return_rlp(io, &rlp, peer, - ChainSync::return_block_headers, + SyncSupplier::return_block_headers, |e| format!("Error sending block headers: {:?}", e)), GET_RECEIPTS_PACKET => ChainSync::return_rlp(io, &rlp, peer, @@ -2456,72 +2387,6 @@ mod tests { assert_eq!(1, io.packets.len()); } - #[test] - fn return_block_headers() { - use ethcore::views::HeaderView; - fn make_hash_req(h: &H256, count: usize, skip: usize, reverse: bool) -> Bytes { - let mut rlp = RlpStream::new_list(4); - rlp.append(h); - rlp.append(&count); - rlp.append(&skip); - rlp.append(&if reverse {1u32} else {0u32}); - rlp.out() - } - - fn make_num_req(n: usize, count: usize, skip: usize, reverse: bool) -> Bytes { - let mut rlp = RlpStream::new_list(4); - rlp.append(&n); - rlp.append(&count); - rlp.append(&skip); - rlp.append(&if reverse {1u32} else {0u32}); - rlp.out() - } - fn to_header_vec(rlp: ::chain::RlpResponseResult) -> Vec { - Rlp::new(&rlp.unwrap().unwrap().1.out()).iter().map(|r| r.as_raw().to_vec()).collect() - } - - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Nothing); - let blocks: Vec<_> = (0 .. 100) - .map(|i| (&client as &BlockChainClient).block(BlockId::Number(i as BlockNumber)).map(|b| b.into_inner()).unwrap()).collect(); - let headers: Vec<_> = blocks.iter().map(|b| Rlp::new(b).at(0).unwrap().as_raw().to_vec()).collect(); - let hashes: Vec<_> = headers.iter().map(|h| view!(HeaderView, h).hash()).collect(); - - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let io = TestIo::new(&mut client, &ss, &queue, None); - - let unknown: H256 = H256::new(); - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, false)), 0); - assert!(to_header_vec(result).is_empty()); - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, true)), 0); - assert!(to_header_vec(result).is_empty()); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, true)), 0); - assert_eq!(to_header_vec(result), vec![headers[2].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, false)), 0); - assert_eq!(to_header_vec(result), vec![headers[2].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, false)), 0); - assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, true)), 0); - assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, true)), 0); - assert_eq!(to_header_vec(result), vec![headers[2].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, false)), 0); - assert_eq!(to_header_vec(result), vec![headers[2].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, false)), 0); - assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]); - - let result = ChainSync::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, true)), 0); - assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]); - } - #[test] fn return_nodes() { let mut client = TestBlockChainClient::new(); diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs new file mode 100644 index 00000000000..2cec10fca88 --- /dev/null +++ b/ethcore/sync/src/chain/supplier.rs @@ -0,0 +1,186 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use bytes::Bytes; +use ethcore::client::{BlockId}; +use ethcore::header::{BlockNumber}; +use ethereum_types::{H256}; +use network::{PeerId}; +use rlp::{Rlp, RlpStream}; +use std::cmp; +use sync_io::SyncIo; + +use super::{ + RlpResponseResult, + BLOCK_HEADERS_PACKET, + MAX_HEADERS_TO_SEND, +}; + +pub struct SyncSupplier {} + +impl SyncSupplier { + /// Respond to GetBlockHeaders request + pub fn return_block_headers(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + // Packet layout: + // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] + let max_headers: usize = r.val_at(1)?; + let skip: usize = r.val_at(2)?; + let reverse: bool = r.val_at(3)?; + let last = io.chain().chain_info().best_block_number; + let number = if r.at(0)?.size() == 32 { + // id is a hash + let hash: H256 = r.val_at(0)?; + trace!(target: "sync", "{} -> GetBlockHeaders (hash: {}, max: {}, skip: {}, reverse:{})", peer_id, hash, max_headers, skip, reverse); + match io.chain().block_header(BlockId::Hash(hash)) { + Some(hdr) => { + let number = hdr.number().into(); + debug_assert_eq!(hdr.hash(), hash); + + if max_headers == 1 || io.chain().block_hash(BlockId::Number(number)) != Some(hash) { + // Non canonical header or single header requested + // TODO: handle single-step reverse hashchains of non-canon hashes + trace!(target:"sync", "Returning single header: {:?}", hash); + let mut rlp = RlpStream::new_list(1); + rlp.append_raw(&hdr.into_inner(), 1); + return Ok(Some((BLOCK_HEADERS_PACKET, rlp))); + } + number + } + None => return Ok(Some((BLOCK_HEADERS_PACKET, RlpStream::new_list(0)))) //no such header, return nothing + } + } else { + trace!(target: "sync", "{} -> GetBlockHeaders (number: {}, max: {}, skip: {}, reverse:{})", peer_id, r.val_at::(0)?, max_headers, skip, reverse); + r.val_at(0)? + }; + + let mut number = if reverse { + cmp::min(last, number) + } else { + cmp::max(0, number) + }; + let max_count = cmp::min(MAX_HEADERS_TO_SEND, max_headers); + let mut count = 0; + let mut data = Bytes::new(); + let inc = (skip + 1) as BlockNumber; + let overlay = io.chain_overlay().read(); + + while number <= last && count < max_count { + if let Some(hdr) = overlay.get(&number) { + trace!(target: "sync", "{}: Returning cached fork header", peer_id); + data.extend_from_slice(hdr); + count += 1; + } else if let Some(hdr) = io.chain().block_header(BlockId::Number(number)) { + data.append(&mut hdr.into_inner()); + count += 1; + } else { + // No required block. + break; + } + if reverse { + if number <= inc || number == 0 { + break; + } + number -= inc; + } + else { + number += inc; + } + } + let mut rlp = RlpStream::new_list(count as usize); + rlp.append_raw(&data, count as usize); + trace!(target: "sync", "{} -> GetBlockHeaders: returned {} entries", peer_id, count); + Ok(Some((BLOCK_HEADERS_PACKET, rlp))) + } +} + +#[cfg(test)] +mod test { + use std::collections::{VecDeque}; + use tests::helpers::{TestIo}; + use tests::snapshot::TestSnapshotService; + use ethereum_types::{H256}; + use parking_lot::RwLock; + use bytes::Bytes; + use rlp::{Rlp, RlpStream}; + use super::*; + use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient}; + + #[test] + fn return_block_headers() { + use ethcore::views::HeaderView; + fn make_hash_req(h: &H256, count: usize, skip: usize, reverse: bool) -> Bytes { + let mut rlp = RlpStream::new_list(4); + rlp.append(h); + rlp.append(&count); + rlp.append(&skip); + rlp.append(&if reverse {1u32} else {0u32}); + rlp.out() + } + + fn make_num_req(n: usize, count: usize, skip: usize, reverse: bool) -> Bytes { + let mut rlp = RlpStream::new_list(4); + rlp.append(&n); + rlp.append(&count); + rlp.append(&skip); + rlp.append(&if reverse {1u32} else {0u32}); + rlp.out() + } + fn to_header_vec(rlp: ::chain::RlpResponseResult) -> Vec { + Rlp::new(&rlp.unwrap().unwrap().1.out()).iter().map(|r| r.as_raw().to_vec()).collect() + } + + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Nothing); + let blocks: Vec<_> = (0 .. 100) + .map(|i| (&client as &BlockChainClient).block(BlockId::Number(i as BlockNumber)).map(|b| b.into_inner()).unwrap()).collect(); + let headers: Vec<_> = blocks.iter().map(|b| Rlp::new(b).at(0).unwrap().as_raw().to_vec()).collect(); + let hashes: Vec<_> = headers.iter().map(|h| view!(HeaderView, h).hash()).collect(); + + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let io = TestIo::new(&mut client, &ss, &queue, None); + + let unknown: H256 = H256::new(); + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, false)), 0); + assert!(to_header_vec(result).is_empty()); + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&unknown, 1, 0, true)), 0); + assert!(to_header_vec(result).is_empty()); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, true)), 0); + assert_eq!(to_header_vec(result), vec![headers[2].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[2], 1, 0, false)), 0); + assert_eq!(to_header_vec(result), vec![headers[2].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, false)), 0); + assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_hash_req(&hashes[50], 3, 5, true)), 0); + assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, true)), 0); + assert_eq!(to_header_vec(result), vec![headers[2].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(2, 1, 0, false)), 0); + assert_eq!(to_header_vec(result), vec![headers[2].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, false)), 0); + assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[56].clone(), headers[62].clone()]); + + let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, true)), 0); + assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]); + } +} From d5bcfa58d68a7bcd2f83f2a78cc5cf35369792a0 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 11:41:48 +0200 Subject: [PATCH 02/16] WIP - updated chain sync supplier --- ethcore/sync/src/chain/mod.rs | 224 +++-------------------------- ethcore/sync/src/chain/supplier.rs | 193 ++++++++++++++++++++++++- 2 files changed, 213 insertions(+), 204 deletions(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index b96574621e2..95d4d98c7cc 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -121,7 +121,7 @@ use self::supplier::SyncSupplier; known_heap_size!(0, PeerInfo); -type PacketDecodeError = DecoderError; +pub type PacketDecodeError = DecoderError; /// 63 version of Ethereum protocol. pub const ETH_PROTOCOL_VERSION_63: u8 = 63; @@ -136,9 +136,9 @@ pub const PAR_PROTOCOL_VERSION_3: u8 = 3; const MAX_BODIES_TO_SEND: usize = 256; pub const MAX_HEADERS_TO_SEND: usize = 512; -const MAX_NODE_DATA_TO_SEND: usize = 1024; -const MAX_RECEIPTS_TO_SEND: usize = 1024; -const MAX_RECEIPTS_HEADERS_TO_SEND: usize = 256; +pub const MAX_NODE_DATA_TO_SEND: usize = 1024; +pub const MAX_RECEIPTS_TO_SEND: usize = 1024; +pub const MAX_RECEIPTS_HEADERS_TO_SEND: usize = 256; const MIN_PEERS_PROPAGATION: usize = 4; const MAX_PEERS_PROPAGATION: usize = 128; const MAX_PEER_LAG_PROPAGATION: BlockNumber = 20; @@ -162,17 +162,17 @@ const GET_BLOCK_BODIES_PACKET: u8 = 0x05; const BLOCK_BODIES_PACKET: u8 = 0x06; const NEW_BLOCK_PACKET: u8 = 0x07; -const GET_NODE_DATA_PACKET: u8 = 0x0d; -const NODE_DATA_PACKET: u8 = 0x0e; -const GET_RECEIPTS_PACKET: u8 = 0x0f; -const RECEIPTS_PACKET: u8 = 0x10; +pub const GET_NODE_DATA_PACKET: u8 = 0x0d; +pub const NODE_DATA_PACKET: u8 = 0x0e; +pub const GET_RECEIPTS_PACKET: u8 = 0x0f; +pub const RECEIPTS_PACKET: u8 = 0x10; pub const ETH_PACKET_COUNT: u8 = 0x11; const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11; -const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12; +pub const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12; const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13; -const SNAPSHOT_DATA_PACKET: u8 = 0x14; +pub const SNAPSHOT_DATA_PACKET: u8 = 0x14; const CONSENSUS_DATA_PACKET: u8 = 0x15; const PRIVATE_TRANSACTION_PACKET: u8 = 0x16; const SIGNED_PRIVATE_TRANSACTION_PACKET: u8 = 0x17; @@ -1580,138 +1580,32 @@ impl ChainSync { Ok(Some((BLOCK_BODIES_PACKET, rlp))) } - /// Respond to GetNodeData request - fn return_node_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let mut count = r.item_count().unwrap_or(0); - trace!(target: "sync", "{} -> GetNodeData: {} entries", peer_id, count); - if count == 0 { - debug!(target: "sync", "Empty GetNodeData request, ignoring."); - return Ok(None); - } - count = cmp::min(count, MAX_NODE_DATA_TO_SEND); - let mut added = 0usize; - let mut data = Vec::new(); - for i in 0..count { - if let Some(node) = io.chain().state_data(&r.val_at::(i)?) { - data.push(node); - added += 1; - } - } - trace!(target: "sync", "{} -> GetNodeData: return {} entries", peer_id, added); - let mut rlp = RlpStream::new_list(added); - for d in data { - rlp.append(&d); - } - Ok(Some((NODE_DATA_PACKET, rlp))) - } - - fn return_receipts(io: &SyncIo, rlp: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let mut count = rlp.item_count().unwrap_or(0); - trace!(target: "sync", "{} -> GetReceipts: {} entries", peer_id, count); - if count == 0 { - debug!(target: "sync", "Empty GetReceipts request, ignoring."); - return Ok(None); - } - count = cmp::min(count, MAX_RECEIPTS_HEADERS_TO_SEND); - let mut added_headers = 0usize; - let mut added_receipts = 0usize; - let mut data = Bytes::new(); - for i in 0..count { - if let Some(mut receipts_bytes) = io.chain().block_receipts(&rlp.val_at::(i)?) { - data.append(&mut receipts_bytes); - added_receipts += receipts_bytes.len(); - added_headers += 1; - if added_receipts > MAX_RECEIPTS_TO_SEND { break; } - } - } - let mut rlp_result = RlpStream::new_list(added_headers); - rlp_result.append_raw(&data, added_headers); - Ok(Some((RECEIPTS_PACKET, rlp_result))) - } - - /// Respond to GetSnapshotManifest request - fn return_snapshot_manifest(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let count = r.item_count().unwrap_or(0); - trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id); - if count != 0 { - debug!(target: "sync", "Invalid GetSnapshotManifest request, ignoring."); - return Ok(None); - } - let rlp = match io.snapshot_service().manifest() { - Some(manifest) => { - trace!(target: "sync", "{} <- SnapshotManifest", peer_id); - let mut rlp = RlpStream::new_list(1); - rlp.append_raw(&manifest.into_rlp(), 1); - rlp - }, - None => { - trace!(target: "sync", "{}: No manifest to return", peer_id); - RlpStream::new_list(0) - } - }; - Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp))) - } - - /// Respond to GetSnapshotData request - fn return_snapshot_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let hash: H256 = r.val_at(0)?; - trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash); - let rlp = match io.snapshot_service().chunk(hash) { - Some(data) => { - let mut rlp = RlpStream::new_list(1); - trace!(target: "sync", "{} <- SnapshotData", peer_id); - rlp.append(&data); - rlp - }, - None => { - RlpStream::new_list(0) - } - }; - Ok(Some((SNAPSHOT_DATA_PACKET, rlp))) - } - - fn return_rlp(io: &mut SyncIo, rlp: &Rlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError> - where FRlp : Fn(&SyncIo, &Rlp, PeerId) -> RlpResponseResult, - FError : FnOnce(network::Error) -> String - { - let response = rlp_func(io, rlp, peer); - match response { - Err(e) => Err(e), - Ok(Some((packet_id, rlp_stream))) => { - io.respond(packet_id, rlp_stream.out()).unwrap_or_else( - |e| debug!(target: "sync", "{:?}", error_func(e))); - Ok(()) - } - _ => Ok(()) - } - } - /// Dispatch incoming requests and responses pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { let rlp = Rlp::new(data); let result = match packet_id { - GET_BLOCK_BODIES_PACKET => ChainSync::return_rlp(io, &rlp, peer, + GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, ChainSync::return_block_bodies, |e| format!("Error sending block bodies: {:?}", e)), - GET_BLOCK_HEADERS_PACKET => ChainSync::return_rlp(io, &rlp, peer, + GET_BLOCK_HEADERS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, SyncSupplier::return_block_headers, |e| format!("Error sending block headers: {:?}", e)), - GET_RECEIPTS_PACKET => ChainSync::return_rlp(io, &rlp, peer, - ChainSync::return_receipts, + GET_RECEIPTS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_receipts, |e| format!("Error sending receipts: {:?}", e)), - GET_NODE_DATA_PACKET => ChainSync::return_rlp(io, &rlp, peer, - ChainSync::return_node_data, + GET_NODE_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_node_data, |e| format!("Error sending nodes: {:?}", e)), - GET_SNAPSHOT_MANIFEST_PACKET => ChainSync::return_rlp(io, &rlp, peer, - ChainSync::return_snapshot_manifest, + GET_SNAPSHOT_MANIFEST_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_snapshot_manifest, |e| format!("Error sending snapshot manifest: {:?}", e)), - GET_SNAPSHOT_DATA_PACKET => ChainSync::return_rlp(io, &rlp, peer, - ChainSync::return_snapshot_data, + GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_snapshot_data, |e| format!("Error sending snapshot data: {:?}", e)), CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), _ => { @@ -2246,7 +2140,7 @@ fn accepts_service_transaction(client_id: &str) -> bool { } #[cfg(test)] -mod tests { +pub mod tests { use std::collections::{HashSet, VecDeque}; use ethkey; use network::PeerId; @@ -2345,81 +2239,7 @@ mod tests { assert!(!sync_status(SyncState::Idle).is_syncing(queue_info(0, 0))); } - #[test] - fn return_receipts_empty() { - let mut client = TestBlockChainClient::new(); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let io = TestIo::new(&mut client, &ss, &queue, None); - - let result = ChainSync::return_receipts(&io, &Rlp::new(&[0xc0]), 0); - - assert!(result.is_ok()); - } - - #[test] - fn return_receipts() { - let mut client = TestBlockChainClient::new(); - let queue = RwLock::new(VecDeque::new()); - let sync = dummy_sync_with_peer(H256::new(), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let mut receipt_list = RlpStream::new_list(4); - receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); - receipt_list.append(&H256::from("ff00000000000000000000000000000000000000000000000000000000000000")); - receipt_list.append(&H256::from("fff0000000000000000000000000000000000000000000000000000000000000")); - receipt_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000")); - - let receipts_request = receipt_list.out(); - // it returns rlp ONLY for hashes started with "f" - let result = ChainSync::return_receipts(&io, &Rlp::new(&receipts_request.clone()), 0); - - assert!(result.is_ok()); - let rlp_result = result.unwrap(); - assert!(rlp_result.is_some()); - - // the length of two rlp-encoded receipts - assert_eq!(603, rlp_result.unwrap().1.out().len()); - - io.sender = Some(2usize); - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, super::GET_RECEIPTS_PACKET, &receipts_request); - assert_eq!(1, io.packets.len()); - } - - #[test] - fn return_nodes() { - let mut client = TestBlockChainClient::new(); - let queue = RwLock::new(VecDeque::new()); - let sync = dummy_sync_with_peer(H256::new(), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let mut node_list = RlpStream::new_list(3); - node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); - node_list.append(&H256::from("ffffffffffffffffffffffffffffffffffffffffffffaaaaaaaaaaaaaaaaaaaa")); - node_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000")); - - let node_request = node_list.out(); - // it returns rlp ONLY for hashes started with "f" - let result = ChainSync::return_node_data(&io, &Rlp::new(&node_request.clone()), 0); - - assert!(result.is_ok()); - let rlp_result = result.unwrap(); - assert!(rlp_result.is_some()); - - // the length of one rlp-encoded hashe - let rlp = rlp_result.unwrap().1.out(); - let rlp = Rlp::new(&rlp); - assert_eq!(Ok(1), rlp.item_count()); - - io.sender = Some(2usize); - - ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, super::GET_NODE_DATA_PACKET, &node_request); - assert_eq!(1, io.packets.len()); - } - - fn dummy_sync_with_peer(peer_latest_hash: H256, client: &BlockChainClient) -> ChainSync { + pub fn dummy_sync_with_peer(peer_latest_hash: H256, client: &BlockChainClient) -> ChainSync { let mut sync = ChainSync::new(SyncConfig::default(), client, Arc::new(NoopPrivateTxHandler)); insert_dummy_peer(&mut sync, 0, peer_latest_hash); sync diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 2cec10fca88..455c1b7a77e 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -18,15 +18,24 @@ use bytes::Bytes; use ethcore::client::{BlockId}; use ethcore::header::{BlockNumber}; use ethereum_types::{H256}; -use network::{PeerId}; +use network::{self, PeerId}; use rlp::{Rlp, RlpStream}; use std::cmp; use sync_io::SyncIo; use super::{ RlpResponseResult, + PacketDecodeError, + BLOCK_HEADERS_PACKET, MAX_HEADERS_TO_SEND, + MAX_NODE_DATA_TO_SEND, + MAX_RECEIPTS_HEADERS_TO_SEND, + MAX_RECEIPTS_TO_SEND, + NODE_DATA_PACKET, + RECEIPTS_PACKET, + SNAPSHOT_DATA_PACKET, + SNAPSHOT_MANIFEST_PACKET, }; pub struct SyncSupplier {} @@ -104,6 +113,112 @@ impl SyncSupplier { trace!(target: "sync", "{} -> GetBlockHeaders: returned {} entries", peer_id, count); Ok(Some((BLOCK_HEADERS_PACKET, rlp))) } + + /// Respond to GetNodeData request + pub fn return_node_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + let mut count = r.item_count().unwrap_or(0); + trace!(target: "sync", "{} -> GetNodeData: {} entries", peer_id, count); + if count == 0 { + debug!(target: "sync", "Empty GetNodeData request, ignoring."); + return Ok(None); + } + count = cmp::min(count, MAX_NODE_DATA_TO_SEND); + let mut added = 0usize; + let mut data = Vec::new(); + for i in 0..count { + if let Some(node) = io.chain().state_data(&r.val_at::(i)?) { + data.push(node); + added += 1; + } + } + trace!(target: "sync", "{} -> GetNodeData: return {} entries", peer_id, added); + let mut rlp = RlpStream::new_list(added); + for d in data { + rlp.append(&d); + } + Ok(Some((NODE_DATA_PACKET, rlp))) + } + + pub fn return_receipts(io: &SyncIo, rlp: &Rlp, peer_id: PeerId) -> RlpResponseResult { + let mut count = rlp.item_count().unwrap_or(0); + trace!(target: "sync", "{} -> GetReceipts: {} entries", peer_id, count); + if count == 0 { + debug!(target: "sync", "Empty GetReceipts request, ignoring."); + return Ok(None); + } + count = cmp::min(count, MAX_RECEIPTS_HEADERS_TO_SEND); + let mut added_headers = 0usize; + let mut added_receipts = 0usize; + let mut data = Bytes::new(); + for i in 0..count { + if let Some(mut receipts_bytes) = io.chain().block_receipts(&rlp.val_at::(i)?) { + data.append(&mut receipts_bytes); + added_receipts += receipts_bytes.len(); + added_headers += 1; + if added_receipts > MAX_RECEIPTS_TO_SEND { break; } + } + } + let mut rlp_result = RlpStream::new_list(added_headers); + rlp_result.append_raw(&data, added_headers); + Ok(Some((RECEIPTS_PACKET, rlp_result))) + } + + /// Respond to GetSnapshotManifest request + pub fn return_snapshot_manifest(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + let count = r.item_count().unwrap_or(0); + trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id); + if count != 0 { + debug!(target: "sync", "Invalid GetSnapshotManifest request, ignoring."); + return Ok(None); + } + let rlp = match io.snapshot_service().manifest() { + Some(manifest) => { + trace!(target: "sync", "{} <- SnapshotManifest", peer_id); + let mut rlp = RlpStream::new_list(1); + rlp.append_raw(&manifest.into_rlp(), 1); + rlp + }, + None => { + trace!(target: "sync", "{}: No manifest to return", peer_id); + RlpStream::new_list(0) + } + }; + Ok(Some((SNAPSHOT_MANIFEST_PACKET, rlp))) + } + + /// Respond to GetSnapshotData request + pub fn return_snapshot_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + let hash: H256 = r.val_at(0)?; + trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash); + let rlp = match io.snapshot_service().chunk(hash) { + Some(data) => { + let mut rlp = RlpStream::new_list(1); + trace!(target: "sync", "{} <- SnapshotData", peer_id); + rlp.append(&data); + rlp + }, + None => { + RlpStream::new_list(0) + } + }; + Ok(Some((SNAPSHOT_DATA_PACKET, rlp))) + } + + pub fn return_rlp(io: &mut SyncIo, rlp: &Rlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError> + where FRlp : Fn(&SyncIo, &Rlp, PeerId) -> RlpResponseResult, + FError : FnOnce(network::Error) -> String + { + let response = rlp_func(io, rlp, peer); + match response { + Err(e) => Err(e), + Ok(Some((packet_id, rlp_stream))) => { + io.respond(packet_id, rlp_stream.out()).unwrap_or_else( + |e| debug!(target: "sync", "{:?}", error_func(e))); + Ok(()) + } + _ => Ok(()) + } + } } #[cfg(test)] @@ -115,7 +230,7 @@ mod test { use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream}; - use super::*; + use super::{*, super::{*, tests::*}}; use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient}; #[test] @@ -183,4 +298,78 @@ mod test { let result = SyncSupplier::return_block_headers(&io, &Rlp::new(&make_num_req(50, 3, 5, true)), 0); assert_eq!(to_header_vec(result), vec![headers[50].clone(), headers[44].clone(), headers[38].clone()]); } + + #[test] + fn return_nodes() { + let mut client = TestBlockChainClient::new(); + let queue = RwLock::new(VecDeque::new()); + let sync = dummy_sync_with_peer(H256::new(), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let mut node_list = RlpStream::new_list(3); + node_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); + node_list.append(&H256::from("ffffffffffffffffffffffffffffffffffffffffffffaaaaaaaaaaaaaaaaaaaa")); + node_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000")); + + let node_request = node_list.out(); + // it returns rlp ONLY for hashes started with "f" + let result = SyncSupplier::return_node_data(&io, &Rlp::new(&node_request.clone()), 0); + + assert!(result.is_ok()); + let rlp_result = result.unwrap(); + assert!(rlp_result.is_some()); + + // the length of one rlp-encoded hashe + let rlp = rlp_result.unwrap().1.out(); + let rlp = Rlp::new(&rlp); + assert_eq!(Ok(1), rlp.item_count()); + + io.sender = Some(2usize); + + ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_NODE_DATA_PACKET, &node_request); + assert_eq!(1, io.packets.len()); + } + + #[test] + fn return_receipts_empty() { + let mut client = TestBlockChainClient::new(); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let io = TestIo::new(&mut client, &ss, &queue, None); + + let result = SyncSupplier::return_receipts(&io, &Rlp::new(&[0xc0]), 0); + + assert!(result.is_ok()); + } + + #[test] + fn return_receipts() { + let mut client = TestBlockChainClient::new(); + let queue = RwLock::new(VecDeque::new()); + let sync = dummy_sync_with_peer(H256::new(), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let mut receipt_list = RlpStream::new_list(4); + receipt_list.append(&H256::from("0000000000000000000000000000000000000000000000005555555555555555")); + receipt_list.append(&H256::from("ff00000000000000000000000000000000000000000000000000000000000000")); + receipt_list.append(&H256::from("fff0000000000000000000000000000000000000000000000000000000000000")); + receipt_list.append(&H256::from("aff0000000000000000000000000000000000000000000000000000000000000")); + + let receipts_request = receipt_list.out(); + // it returns rlp ONLY for hashes started with "f" + let result = SyncSupplier::return_receipts(&io, &Rlp::new(&receipts_request.clone()), 0); + + assert!(result.is_ok()); + let rlp_result = result.unwrap(); + assert!(rlp_result.is_some()); + + // the length of two rlp-encoded receipts + assert_eq!(603, rlp_result.unwrap().1.out().len()); + + io.sender = Some(2usize); + ChainSync::dispatch_packet(&RwLock::new(sync), &mut io, 0usize, GET_RECEIPTS_PACKET, &receipts_request); + assert_eq!(1, io.packets.len()); + } } From 8217b8229c83057af7500f863fc7ca6ecac55657 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 11:51:01 +0200 Subject: [PATCH 03/16] Finish refactoring the Chain Sync Supplier --- ethcore/sync/src/chain/mod.rs | 69 +++-------------------- ethcore/sync/src/chain/supplier.rs | 88 +++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 69 deletions(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 95d4d98c7cc..6c0378bacf0 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -134,7 +134,7 @@ pub const PAR_PROTOCOL_VERSION_2: u8 = 2; /// 3 version of Parity protocol (private transactions messages added). pub const PAR_PROTOCOL_VERSION_3: u8 = 3; -const MAX_BODIES_TO_SEND: usize = 256; +pub const MAX_BODIES_TO_SEND: usize = 256; pub const MAX_HEADERS_TO_SEND: usize = 512; pub const MAX_NODE_DATA_TO_SEND: usize = 1024; pub const MAX_RECEIPTS_TO_SEND: usize = 1024; @@ -156,9 +156,9 @@ const SNAPSHOT_MIN_PEERS: usize = 3; const STATUS_PACKET: u8 = 0x00; const NEW_BLOCK_HASHES_PACKET: u8 = 0x01; const TRANSACTIONS_PACKET: u8 = 0x02; -const GET_BLOCK_HEADERS_PACKET: u8 = 0x03; +pub const GET_BLOCK_HEADERS_PACKET: u8 = 0x03; pub const BLOCK_HEADERS_PACKET: u8 = 0x04; -const GET_BLOCK_BODIES_PACKET: u8 = 0x05; +pub const GET_BLOCK_BODIES_PACKET: u8 = 0x05; const BLOCK_BODIES_PACKET: u8 = 0x06; const NEW_BLOCK_PACKET: u8 = 0x07; @@ -169,11 +169,11 @@ pub const RECEIPTS_PACKET: u8 = 0x10; pub const ETH_PACKET_COUNT: u8 = 0x11; -const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11; +pub const GET_SNAPSHOT_MANIFEST_PACKET: u8 = 0x11; pub const SNAPSHOT_MANIFEST_PACKET: u8 = 0x12; -const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13; +pub const GET_SNAPSHOT_DATA_PACKET: u8 = 0x13; pub const SNAPSHOT_DATA_PACKET: u8 = 0x14; -const CONSENSUS_DATA_PACKET: u8 = 0x15; +pub const CONSENSUS_DATA_PACKET: u8 = 0x15; const PRIVATE_TRANSACTION_PACKET: u8 = 0x16; const SIGNED_PRIVATE_TRANSACTION_PACKET: u8 = 0x17; @@ -1558,64 +1558,9 @@ impl ChainSync { io.respond(STATUS_PACKET, packet.out()) } - /// Respond to GetBlockBodies request - fn return_block_bodies(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { - let mut count = r.item_count().unwrap_or(0); - if count == 0 { - debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); - return Ok(None); - } - count = cmp::min(count, MAX_BODIES_TO_SEND); - let mut added = 0usize; - let mut data = Bytes::new(); - for i in 0..count { - if let Some(body) = io.chain().block_body(BlockId::Hash(r.val_at::(i)?)) { - data.append(&mut body.into_inner()); - added += 1; - } - } - let mut rlp = RlpStream::new_list(added); - rlp.append_raw(&data, added); - trace!(target: "sync", "{} -> GetBlockBodies: returned {} entries", peer_id, added); - Ok(Some((BLOCK_BODIES_PACKET, rlp))) - } - /// Dispatch incoming requests and responses pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - let rlp = Rlp::new(data); - let result = match packet_id { - GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - ChainSync::return_block_bodies, - |e| format!("Error sending block bodies: {:?}", e)), - - GET_BLOCK_HEADERS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - SyncSupplier::return_block_headers, - |e| format!("Error sending block headers: {:?}", e)), - - GET_RECEIPTS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - SyncSupplier::return_receipts, - |e| format!("Error sending receipts: {:?}", e)), - - GET_NODE_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - SyncSupplier::return_node_data, - |e| format!("Error sending nodes: {:?}", e)), - - GET_SNAPSHOT_MANIFEST_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - SyncSupplier::return_snapshot_manifest, - |e| format!("Error sending snapshot manifest: {:?}", e)), - - GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, - SyncSupplier::return_snapshot_data, - |e| format!("Error sending snapshot data: {:?}", e)), - CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), - _ => { - sync.write().on_packet(io, peer, packet_id, data); - Ok(()) - } - }; - result.unwrap_or_else(|e| { - debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e); - }) + SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) } pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 455c1b7a77e..74711b51659 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -19,15 +19,29 @@ use ethcore::client::{BlockId}; use ethcore::header::{BlockNumber}; use ethereum_types::{H256}; use network::{self, PeerId}; +use parking_lot::RwLock; use rlp::{Rlp, RlpStream}; use std::cmp; use sync_io::SyncIo; use super::{ + ChainSync, + RlpResponseResult, PacketDecodeError, + BLOCK_BODIES_PACKET, BLOCK_HEADERS_PACKET, + + CONSENSUS_DATA_PACKET, + GET_BLOCK_BODIES_PACKET, + GET_BLOCK_HEADERS_PACKET, + GET_NODE_DATA_PACKET, + GET_RECEIPTS_PACKET, + GET_SNAPSHOT_DATA_PACKET, + GET_SNAPSHOT_MANIFEST_PACKET, + + MAX_BODIES_TO_SEND, MAX_HEADERS_TO_SEND, MAX_NODE_DATA_TO_SEND, MAX_RECEIPTS_HEADERS_TO_SEND, @@ -41,8 +55,46 @@ use super::{ pub struct SyncSupplier {} impl SyncSupplier { + /// Dispatch incoming requests and responses + pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + let rlp = Rlp::new(data); + let result = match packet_id { + GET_BLOCK_BODIES_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_block_bodies, + |e| format!("Error sending block bodies: {:?}", e)), + + GET_BLOCK_HEADERS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_block_headers, + |e| format!("Error sending block headers: {:?}", e)), + + GET_RECEIPTS_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_receipts, + |e| format!("Error sending receipts: {:?}", e)), + + GET_NODE_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_node_data, + |e| format!("Error sending nodes: {:?}", e)), + + GET_SNAPSHOT_MANIFEST_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_snapshot_manifest, + |e| format!("Error sending snapshot manifest: {:?}", e)), + + GET_SNAPSHOT_DATA_PACKET => SyncSupplier::return_rlp(io, &rlp, peer, + SyncSupplier::return_snapshot_data, + |e| format!("Error sending snapshot data: {:?}", e)), + CONSENSUS_DATA_PACKET => ChainSync::on_consensus_packet(io, peer, &rlp), + _ => { + sync.write().on_packet(io, peer, packet_id, data); + Ok(()) + } + }; + result.unwrap_or_else(|e| { + debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e); + }) + } + /// Respond to GetBlockHeaders request - pub fn return_block_headers(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + fn return_block_headers(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { // Packet layout: // [ block: { P , B_32 }, maxHeaders: P, skip: P, reverse: P in { 0 , 1 } ] let max_headers: usize = r.val_at(1)?; @@ -114,8 +166,30 @@ impl SyncSupplier { Ok(Some((BLOCK_HEADERS_PACKET, rlp))) } + /// Respond to GetBlockBodies request + fn return_block_bodies(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + let mut count = r.item_count().unwrap_or(0); + if count == 0 { + debug!(target: "sync", "Empty GetBlockBodies request, ignoring."); + return Ok(None); + } + count = cmp::min(count, MAX_BODIES_TO_SEND); + let mut added = 0usize; + let mut data = Bytes::new(); + for i in 0..count { + if let Some(body) = io.chain().block_body(BlockId::Hash(r.val_at::(i)?)) { + data.append(&mut body.into_inner()); + added += 1; + } + } + let mut rlp = RlpStream::new_list(added); + rlp.append_raw(&data, added); + trace!(target: "sync", "{} -> GetBlockBodies: returned {} entries", peer_id, added); + Ok(Some((BLOCK_BODIES_PACKET, rlp))) + } + /// Respond to GetNodeData request - pub fn return_node_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + fn return_node_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { let mut count = r.item_count().unwrap_or(0); trace!(target: "sync", "{} -> GetNodeData: {} entries", peer_id, count); if count == 0 { @@ -139,7 +213,7 @@ impl SyncSupplier { Ok(Some((NODE_DATA_PACKET, rlp))) } - pub fn return_receipts(io: &SyncIo, rlp: &Rlp, peer_id: PeerId) -> RlpResponseResult { + fn return_receipts(io: &SyncIo, rlp: &Rlp, peer_id: PeerId) -> RlpResponseResult { let mut count = rlp.item_count().unwrap_or(0); trace!(target: "sync", "{} -> GetReceipts: {} entries", peer_id, count); if count == 0 { @@ -164,7 +238,7 @@ impl SyncSupplier { } /// Respond to GetSnapshotManifest request - pub fn return_snapshot_manifest(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + fn return_snapshot_manifest(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { let count = r.item_count().unwrap_or(0); trace!(target: "sync", "{} -> GetSnapshotManifest", peer_id); if count != 0 { @@ -187,7 +261,7 @@ impl SyncSupplier { } /// Respond to GetSnapshotData request - pub fn return_snapshot_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { + fn return_snapshot_data(io: &SyncIo, r: &Rlp, peer_id: PeerId) -> RlpResponseResult { let hash: H256 = r.val_at(0)?; trace!(target: "sync", "{} -> GetSnapshotData {:?}", peer_id, hash); let rlp = match io.snapshot_service().chunk(hash) { @@ -204,7 +278,7 @@ impl SyncSupplier { Ok(Some((SNAPSHOT_DATA_PACKET, rlp))) } - pub fn return_rlp(io: &mut SyncIo, rlp: &Rlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError> + fn return_rlp(io: &mut SyncIo, rlp: &Rlp, peer: PeerId, rlp_func: FRlp, error_func: FError) -> Result<(), PacketDecodeError> where FRlp : Fn(&SyncIo, &Rlp, PeerId) -> RlpResponseResult, FError : FnOnce(network::Error) -> String { @@ -230,7 +304,7 @@ mod test { use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream}; - use super::{*, super::{*, tests::*}}; + use super::{*, super::tests::*}; use ethcore::client::{BlockChainClient, EachBlockWith, TestBlockChainClient}; #[test] From da0eb24a19647e9faf2471020e0a2f7b22257c9e Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 12:40:00 +0200 Subject: [PATCH 04/16] Create Chain Sync Requester --- ethcore/sync/src/chain/mod.rs | 141 +++---------------------- ethcore/sync/src/chain/requester.rs | 155 ++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 125 deletions(-) create mode 100644 ethcore/sync/src/chain/requester.rs diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 6c0378bacf0..82145faac0f 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -88,6 +88,7 @@ /// All other messages are ignored. /// +mod requester; mod supplier; use std::sync::Arc; @@ -109,7 +110,7 @@ use ethcore::snapshot::{ManifestData, RestorationStatus}; use transaction::SignedTransaction; use sync_io::SyncIo; use super::{WarpSync, SyncConfig}; -use block_sync::{BlockDownloader, BlockRequest, BlockDownloaderImportError as DownloaderImportError, DownloadAction}; +use block_sync::{BlockDownloader, BlockDownloaderImportError as DownloaderImportError, DownloadAction}; use rand::Rng; use snapshot::{Snapshot, ChunkType}; use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID}; @@ -118,6 +119,7 @@ use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; use self::supplier::SyncSupplier; +use self::requester::SyncRequester; known_heap_size!(0, PeerInfo); @@ -271,7 +273,7 @@ impl SyncStatus { #[derive(PartialEq, Eq, Debug, Clone)] /// Peer data type requested -enum PeerAsking { +pub enum PeerAsking { Nothing, ForkHeader, BlockHeaders, @@ -283,7 +285,7 @@ enum PeerAsking { #[derive(PartialEq, Eq, Debug, Clone, Copy)] /// Block downloader channel. -enum BlockSet { +pub enum BlockSet { /// New blocks better than out best blocks NewBlocks, /// Missing old blocks @@ -301,7 +303,7 @@ enum ForkConfirmation { #[derive(Clone)] /// Syncing peer information -struct PeerInfo { +pub struct PeerInfo { /// eth protocol version protocol_version: u8, /// Peer chain genesis hash @@ -366,6 +368,9 @@ mod random { pub fn new() -> rand::XorShiftRng { rand::XorShiftRng::from_seed([0, 1, 2, 3]) } } +pub type RlpResponseResult = Result, PacketDecodeError>; +pub type Peers = HashMap; + /// Blockchain sync handler. /// See module documentation for more details. pub struct ChainSync { @@ -376,7 +381,7 @@ pub struct ChainSync { /// Highest block number seen highest_block: Option, /// All connected peers - peers: HashMap, + peers: Peers, /// Peers active for current sync round active_peers: HashSet, /// Block download process for new blocks @@ -406,8 +411,6 @@ pub struct ChainSync { warp_sync: WarpSync, } -pub type RlpResponseResult = Result, PacketDecodeError>; - impl ChainSync { /// Create a new instance of syncing strategy. pub fn new(config: SyncConfig, chain: &BlockChainClient, private_tx_handler: Arc) -> ChainSync { @@ -598,7 +601,7 @@ impl ChainSync { if !self.snapshot.have_manifest() { for p in peers { if self.peers.get(p).map_or(false, |p| p.asking == PeerAsking::Nothing) { - self.request_snapshot_manifest(io, *p); + SyncRequester::request_snapshot_manifest(&mut self.peers, io, *p); } } self.state = SyncState::SnapshotManifest; @@ -698,7 +701,7 @@ impl ChainSync { self.active_peers.insert(peer_id.clone()); debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); if let Some((fork_block, _)) = self.fork_block { - self.request_fork_header_by_number(io, peer_id, fork_block); + SyncRequester::request_fork_header_by_number(&mut self.peers, io, peer_id, fork_block); } else { self.sync_peer(io, peer_id, false); } @@ -1280,7 +1283,7 @@ impl ChainSync { // check if got new blocks to download trace!(target: "sync", "Syncing with peer {}, force={}, td={:?}, our td={}, state={:?}", peer_id, force, peer_difficulty, syncing_difficulty, self.state); if let Some(request) = self.new_blocks.request_blocks(io, num_active_peers) { - self.request_blocks(io, peer_id, request, BlockSet::NewBlocks); + SyncRequester::request_blocks(&mut self.peers, io, peer_id, request, BlockSet::NewBlocks); if self.state == SyncState::Idle { self.state = SyncState::Blocks; } @@ -1289,7 +1292,7 @@ impl ChainSync { } if let Some(request) = self.old_blocks.as_mut().and_then(|d| d.request_blocks(io, num_active_peers)) { - self.request_blocks(io, peer_id, request, BlockSet::OldBlocks); + SyncRequester::request_blocks(&mut self.peers, io, peer_id, request, BlockSet::OldBlocks); return; } }, @@ -1302,7 +1305,8 @@ impl ChainSync { } } if peer_snapshot_hash.is_some() && peer_snapshot_hash == self.snapshot.snapshot_hash() { - self.request_snapshot_data(io, peer_id); + self.clear_peer_download(peer_id); + SyncRequester::request_snapshot_data(&mut self.peers, &mut self.snapshot, io, peer_id); } }, SyncState::SnapshotManifest | //already downloading from other peer @@ -1313,33 +1317,6 @@ impl ChainSync { } } - /// Perofrm block download request` - fn request_blocks(&mut self, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) { - match request { - BlockRequest::Headers { start, count, skip } => { - self.request_headers_by_hash(io, peer_id, &start, count, skip, false, block_set); - }, - BlockRequest::Bodies { hashes } => { - self.request_bodies(io, peer_id, hashes, block_set); - }, - BlockRequest::Receipts { hashes } => { - self.request_receipts(io, peer_id, hashes, block_set); - }, - } - } - - /// Find some headers or blocks to download for a peer. - fn request_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId) { - self.clear_peer_download(peer_id); - // find chunk data to download - if let Some(hash) = self.snapshot.needed_chunk() { - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - peer.asking_snapshot_data = Some(hash.clone()); - } - self.request_snapshot_chunk(io, peer_id, &hash); - } - } - /// Clear all blocks/headers marked as being downloaded by a peer. fn clear_peer_download(&mut self, peer_id: PeerId) { if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { @@ -1393,72 +1370,6 @@ impl ChainSync { } } - /// Request headers from a peer by block hash - fn request_headers_by_hash(&mut self, sync: &mut SyncIo, peer_id: PeerId, h: &H256, count: u64, skip: u64, reverse: bool, set: BlockSet) { - trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}, set = {:?}", peer_id, count, h, set); - let mut rlp = RlpStream::new_list(4); - rlp.append(h); - rlp.append(&count); - rlp.append(&skip); - rlp.append(&if reverse {1u32} else {0u32}); - self.send_request(sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); - let peer = self.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); - peer.asking_hash = Some(h.clone()); - peer.block_set = Some(set); - } - - /// Request headers from a peer by block number - fn request_fork_header_by_number(&mut self, sync: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { - trace!(target: "sync", "{} <- GetForkHeader: at {}", peer_id, n); - let mut rlp = RlpStream::new_list(4); - rlp.append(&n); - rlp.append(&1u32); - rlp.append(&0u32); - rlp.append(&0u32); - self.send_request(sync, peer_id, PeerAsking::ForkHeader, GET_BLOCK_HEADERS_PACKET, rlp.out()); - } - - /// Request snapshot manifest from a peer. - fn request_snapshot_manifest(&mut self, sync: &mut SyncIo, peer_id: PeerId) { - trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id); - let rlp = RlpStream::new_list(0); - self.send_request(sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out()); - } - - /// Request snapshot chunk from a peer. - fn request_snapshot_chunk(&mut self, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) { - trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk); - let mut rlp = RlpStream::new_list(1); - rlp.append(chunk); - self.send_request(sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out()); - } - - /// Request block bodies from a peer - fn request_bodies(&mut self, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { - let mut rlp = RlpStream::new_list(hashes.len()); - trace!(target: "sync", "{} <- GetBlockBodies: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); - for h in &hashes { - rlp.append(&h.clone()); - } - self.send_request(sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); - let peer = self.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); - peer.asking_blocks = hashes; - peer.block_set = Some(set); - } - - /// Request block receipts from a peer - fn request_receipts(&mut self, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { - let mut rlp = RlpStream::new_list(hashes.len()); - trace!(target: "sync", "{} <- GetBlockReceipts: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); - for h in &hashes { - rlp.append(&h.clone()); - } - self.send_request(sync, peer_id, PeerAsking::BlockReceipts, GET_RECEIPTS_PACKET, rlp.out()); - let peer = self.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); - peer.asking_blocks = hashes; - peer.block_set = Some(set); - } - /// Reset peer status after request is complete. fn reset_peer_asking(&mut self, peer_id: PeerId, asking: PeerAsking) -> bool { if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { @@ -1476,26 +1387,6 @@ impl ChainSync { false } - /// Generic request sender - fn send_request(&mut self, sync: &mut SyncIo, peer_id: PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - if peer.asking != PeerAsking::Nothing { - warn!(target:"sync", "Asking {:?} while requesting {:?}", peer.asking, asking); - } - peer.asking = asking; - peer.ask_time = Instant::now(); - let result = if packet_id >= ETH_PACKET_COUNT { - sync.send_protocol(WARP_SYNC_PROTOCOL_ID, peer_id, packet_id, packet) - } else { - sync.send(peer_id, packet_id, packet) - }; - if let Err(e) = result { - debug!(target:"sync", "Error sending request: {:?}", e); - sync.disconnect_peer(peer_id); - } - } - } - /// Generic packet sender fn send_packet(&mut self, sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { if let Err(e) = sync.send(peer_id, packet_id, packet) { diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs new file mode 100644 index 00000000000..506aec92fc1 --- /dev/null +++ b/ethcore/sync/src/chain/requester.rs @@ -0,0 +1,155 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use api::{WARP_SYNC_PROTOCOL_ID}; +use block_sync::{BlockRequest}; +use bytes::Bytes; +use ethcore::header::{BlockNumber}; +use ethereum_types::{H256}; +use network::{PeerId, PacketId}; +use rlp::{RlpStream}; +use snapshot::{Snapshot}; +use std::time::{Instant}; +use sync_io::SyncIo; + +use super::{ + BlockSet, + PeerAsking, + Peers, + + ETH_PACKET_COUNT, + GET_BLOCK_BODIES_PACKET, + GET_BLOCK_HEADERS_PACKET, + GET_RECEIPTS_PACKET, + GET_SNAPSHOT_DATA_PACKET, + GET_SNAPSHOT_MANIFEST_PACKET, +}; + +pub struct SyncRequester {} + +impl SyncRequester { + /// Perofrm block download request` + pub fn request_blocks(peers: &mut Peers, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) { + match request { + BlockRequest::Headers { start, count, skip } => { + SyncRequester::request_headers_by_hash(peers, io, peer_id, &start, count, skip, false, block_set); + }, + BlockRequest::Bodies { hashes } => { + SyncRequester::request_bodies(peers, io, peer_id, hashes, block_set); + }, + BlockRequest::Receipts { hashes } => { + SyncRequester::request_receipts(peers, io, peer_id, hashes, block_set); + }, + } + } + + /// Request block bodies from a peer + fn request_bodies(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { + let mut rlp = RlpStream::new_list(hashes.len()); + trace!(target: "sync", "{} <- GetBlockBodies: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); + for h in &hashes { + rlp.append(&h.clone()); + } + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); + let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + peer.asking_blocks = hashes; + peer.block_set = Some(set); + } + + /// Request headers from a peer by block number + pub fn request_fork_header_by_number(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { + trace!(target: "sync", "{} <- GetForkHeader: at {}", peer_id, n); + let mut rlp = RlpStream::new_list(4); + rlp.append(&n); + rlp.append(&1u32); + rlp.append(&0u32); + rlp.append(&0u32); + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::ForkHeader, GET_BLOCK_HEADERS_PACKET, rlp.out()); + } + + /// Find some headers or blocks to download for a peer. + pub fn request_snapshot_data(peers: &mut Peers, snapshot: &mut Snapshot, io: &mut SyncIo, peer_id: PeerId) { + // find chunk data to download + if let Some(hash) = snapshot.needed_chunk() { + if let Some(ref mut peer) = peers.get_mut(&peer_id) { + peer.asking_snapshot_data = Some(hash.clone()); + } + SyncRequester::request_snapshot_chunk(peers, io, peer_id, &hash); + } + } + + /// Request snapshot manifest from a peer. + pub fn request_snapshot_manifest(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId) { + trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id); + let rlp = RlpStream::new_list(0); + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out()); + } + + /// Request headers from a peer by block hash + fn request_headers_by_hash(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, h: &H256, count: u64, skip: u64, reverse: bool, set: BlockSet) { + trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}, set = {:?}", peer_id, count, h, set); + let mut rlp = RlpStream::new_list(4); + rlp.append(h); + rlp.append(&count); + rlp.append(&skip); + rlp.append(&if reverse {1u32} else {0u32}); + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); + let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + peer.asking_hash = Some(h.clone()); + peer.block_set = Some(set); + } + + /// Request block receipts from a peer + fn request_receipts(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { + let mut rlp = RlpStream::new_list(hashes.len()); + trace!(target: "sync", "{} <- GetBlockReceipts: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); + for h in &hashes { + rlp.append(&h.clone()); + } + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockReceipts, GET_RECEIPTS_PACKET, rlp.out()); + let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + peer.asking_blocks = hashes; + peer.block_set = Some(set); + } + + /// Request snapshot chunk from a peer. + fn request_snapshot_chunk(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) { + trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk); + let mut rlp = RlpStream::new_list(1); + rlp.append(chunk); + SyncRequester::send_request(peers, sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out()); + } + + /// Generic request sender + fn send_request(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { + if let Some(ref mut peer) = peers.get_mut(&peer_id) { + if peer.asking != PeerAsking::Nothing { + warn!(target:"sync", "Asking {:?} while requesting {:?}", peer.asking, asking); + } + peer.asking = asking; + peer.ask_time = Instant::now(); + let result = if packet_id >= ETH_PACKET_COUNT { + sync.send_protocol(WARP_SYNC_PROTOCOL_ID, peer_id, packet_id, packet) + } else { + sync.send(peer_id, packet_id, packet) + }; + if let Err(e) = result { + debug!(target:"sync", "Error sending request: {:?}", e); + sync.disconnect_peer(peer_id); + } + } + } +} From 017fd34392d44e44039e77e8afa108a0ef5fee78 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 15:21:46 +0200 Subject: [PATCH 05/16] Add Propagator for Chain Sync --- ethcore/sync/src/chain/mod.rs | 347 ++++----------------------- ethcore/sync/src/chain/propagator.rs | 336 ++++++++++++++++++++++++++ 2 files changed, 378 insertions(+), 305 deletions(-) create mode 100644 ethcore/sync/src/chain/propagator.rs diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 82145faac0f..c02759b0126 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -88,6 +88,7 @@ /// All other messages are ignored. /// +mod propagator; mod requester; mod supplier; @@ -101,13 +102,12 @@ use ethereum_types::{H256, U256}; use plain_hasher::H256FastMap; use parking_lot::RwLock; use bytes::Bytes; -use rlp::{Rlp, RlpStream, DecoderError, Encodable}; +use rlp::{Rlp, RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockImportError, BlockImportErrorKind, BlockQueueInfo}; use ethcore::error::*; use ethcore::snapshot::{ManifestData, RestorationStatus}; -use transaction::SignedTransaction; use sync_io::SyncIo; use super::{WarpSync, SyncConfig}; use block_sync::{BlockDownloader, BlockDownloaderImportError as DownloaderImportError, DownloadAction}; @@ -118,8 +118,9 @@ use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; -use self::supplier::SyncSupplier; +use self::propagator::SyncPropagator; use self::requester::SyncRequester; +use self::supplier::SyncSupplier; known_heap_size!(0, PeerInfo); @@ -358,12 +359,12 @@ impl PeerInfo { } #[cfg(not(test))] -mod random { +pub mod random { use rand; pub fn new() -> rand::ThreadRng { rand::thread_rng() } } #[cfg(test)] -mod random { +pub mod random { use rand::{self, SeedableRng}; pub fn new() -> rand::XorShiftRng { rand::XorShiftRng::from_seed([0, 1, 2, 3]) } } @@ -1387,14 +1388,6 @@ impl ChainSync { false } - /// Generic packet sender - fn send_packet(&mut self, sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { - if let Err(e) = sync.send(peer_id, packet_id, packet) { - debug!(target:"sync", "Error sending packet: {:?}", e); - sync.disconnect_peer(peer_id); - } - } - /// Called when peer sends us new transactions fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { // Accept transactions only when fully synced @@ -1632,236 +1625,6 @@ impl ChainSync { self.peers.iter().filter_map(|(id, p)| if p.protocol_version >= PAR_PROTOCOL_VERSION_3 { Some(*id) } else { None }).collect() } - /// propagates latest block to a set of peers - fn propagate_blocks(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize { - trace!(target: "sync", "Sending NewBlocks to {:?}", peers); - let mut sent = 0; - for peer_id in peers { - if blocks.is_empty() { - let rlp = ChainSync::create_latest_block_rlp(io.chain()); - self.send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); - } else { - for h in blocks { - let rlp = ChainSync::create_new_block_rlp(io.chain(), h); - self.send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); - } - } - if let Some(ref mut peer) = self.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); - } - sent += 1; - } - sent - } - - /// propagates new known hashes to all peers - fn propagate_new_hashes(&mut self, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize { - trace!(target: "sync", "Sending NewHashes to {:?}", peers); - let mut sent = 0; - let last_parent = *io.chain().best_block_header().parent_hash(); - for peer_id in peers { - sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) { - Some(rlp) => { - { - if let Some(ref mut peer) = self.peers.get_mut(peer_id) { - peer.latest_hash = chain_info.best_block_hash.clone(); - } - } - self.send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp); - 1 - }, - None => 0 - } - } - sent - } - - /// propagates new transactions to all peers - pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize { - // Early out if nobody to send to. - if self.peers.is_empty() { - return 0; - } - - let transactions = io.chain().ready_transactions(); - if transactions.is_empty() { - return 0; - } - - let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter() - .map(|tx| tx.signed()) - .partition(|tx| !tx.gas_price.is_zero()); - - // usual transactions could be propagated to all peers - let mut affected_peers = HashSet::new(); - if !transactions.is_empty() { - let peers = self.select_peers_for_transactions(|_| true); - affected_peers = self.propagate_transactions_to_peers(io, peers, transactions); - } - - // most of times service_transactions will be empty - // => there's no need to merge packets - if !service_transactions.is_empty() { - let service_transactions_peers = self.select_peers_for_transactions(|peer_id| accepts_service_transaction(&io.peer_info(*peer_id))); - let service_transactions_affected_peers = self.propagate_transactions_to_peers(io, service_transactions_peers, service_transactions); - affected_peers.extend(&service_transactions_affected_peers); - } - - affected_peers.len() - } - - fn select_peers_for_transactions(&self, filter: F) -> Vec - where F: Fn(&PeerId) -> bool { - // sqrt(x)/x scaled to max u32 - let fraction = ((self.peers.len() as f64).powf(-0.5) * (u32::max_value() as f64).round()) as u32; - let small = self.peers.len() < MIN_PEERS_PROPAGATION; - - let mut random = random::new(); - self.peers.keys() - .cloned() - .filter(filter) - .filter(|_| small || random.next_u32() < fraction) - .take(MAX_PEERS_PROPAGATION) - .collect() - } - - fn propagate_transactions_to_peers(&mut self, io: &mut SyncIo, peers: Vec, transactions: Vec<&SignedTransaction>) -> HashSet { - let all_transactions_hashes = transactions.iter() - .map(|tx| tx.hash()) - .collect::>(); - let all_transactions_rlp = { - let mut packet = RlpStream::new_list(transactions.len()); - for tx in &transactions { packet.append(&**tx); } - packet.out() - }; - - // Clear old transactions from stats - self.transactions_stats.retain(&all_transactions_hashes); - - // sqrt(x)/x scaled to max u32 - let block_number = io.chain().chain_info().best_block_number; - - let lucky_peers = { - peers.into_iter() - .filter_map(|peer_id| { - let stats = &mut self.transactions_stats; - let peer_info = self.peers.get_mut(&peer_id) - .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); - - // Send all transactions - if peer_info.last_sent_transactions.is_empty() { - // update stats - for hash in &all_transactions_hashes { - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - stats.propagated(hash, id, block_number); - } - peer_info.last_sent_transactions = all_transactions_hashes.clone(); - return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone())); - } - - // Get hashes of all transactions to send to this peer - let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) - .take(MAX_TRANSACTIONS_TO_PROPAGATE) - .cloned() - .collect::>(); - if to_send.is_empty() { - return None; - } - - // Construct RLP - let (packet, to_send) = { - let mut to_send = to_send; - let mut packet = RlpStream::new(); - packet.begin_unbounded_list(); - let mut pushed = 0; - for tx in &transactions { - let hash = tx.hash(); - if to_send.contains(&hash) { - let mut transaction = RlpStream::new(); - tx.rlp_append(&mut transaction); - let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); - if !appended { - // Maximal packet size reached just proceed with sending - debug!("Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); - to_send = to_send.into_iter().take(pushed).collect(); - break; - } - pushed += 1; - } - } - packet.complete_unbounded_list(); - (packet, to_send) - }; - - // Update stats - let id = io.peer_session_info(peer_id).and_then(|info| info.id); - for hash in &to_send { - // update stats - stats.propagated(hash, id, block_number); - } - - peer_info.last_sent_transactions = all_transactions_hashes - .intersection(&peer_info.last_sent_transactions) - .chain(&to_send) - .cloned() - .collect(); - Some((peer_id, to_send.len(), packet.out())) - }) - .collect::>() - }; - - // Send RLPs - let mut peers = HashSet::new(); - if lucky_peers.len() > 0 { - let mut max_sent = 0; - let lucky_peers_len = lucky_peers.len(); - for (peer_id, sent, rlp) in lucky_peers { - peers.insert(peer_id); - self.send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); - trace!(target: "sync", "{:02} <- Transactions ({} entries)", peer_id, sent); - max_sent = cmp::max(max_sent, sent); - } - debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len); - } - - peers - } - - fn propagate_latest_blocks(&mut self, io: &mut SyncIo, sealed: &[H256]) { - let chain_info = io.chain().chain_info(); - if (((chain_info.best_block_number as i64) - (self.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { - let mut peers = self.get_lagging_peers(&chain_info); - if sealed.is_empty() { - let hashes = self.propagate_new_hashes(&chain_info, io, &peers); - peers = ChainSync::select_random_peers(&peers); - let blocks = self.propagate_blocks(&chain_info, io, sealed, &peers); - if blocks != 0 || hashes != 0 { - trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); - } - } else { - self.propagate_blocks(&chain_info, io, sealed, &peers); - self.propagate_new_hashes(&chain_info, io, &peers); - trace!(target: "sync", "Sent sealed block to all peers"); - }; - } - self.last_sent_block_number = chain_info.best_block_number; - } - - /// Distribute valid proposed blocks to subset of current peers. - fn propagate_proposed_blocks(&mut self, io: &mut SyncIo, proposed: &[Bytes]) { - let peers = self.get_consensus_peers(); - trace!(target: "sync", "Sending proposed blocks to {:?}", peers); - for block in proposed { - let rlp = ChainSync::create_block_rlp( - block, - io.chain().chain_info().total_difficulty - ); - for peer_id in &peers { - self.send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone()); - } - } - } - /// Maintain other peers. Send out any new blocks and transactions pub fn maintain_sync(&mut self, io: &mut SyncIo) { self.maybe_start_snapshot_sync(io); @@ -1875,8 +1638,8 @@ impl ChainSync { if !is_syncing || !sealed.is_empty() || !proposed.is_empty() { trace!(target: "sync", "Propagating blocks, state={:?}", self.state); - self.propagate_latest_blocks(io, sealed); - self.propagate_proposed_blocks(io, proposed); + SyncPropagator::propagate_latest_blocks(self, io, sealed); + SyncPropagator::propagate_proposed_blocks(self, io, proposed); } if !invalid.is_empty() { trace!(target: "sync", "Bad blocks in the queue, restarting"); @@ -1900,15 +1663,6 @@ impl ChainSync { Ok(()) } - /// Broadcast consensus message to peers. - pub fn propagate_consensus_packet(&mut self, io: &mut SyncIo, packet: Bytes) { - let lucky_peers = ChainSync::select_random_peers(&self.get_consensus_peers()); - trace!(target: "sync", "Sending consensus packet to {:?}", lucky_peers); - for peer_id in lucky_peers { - self.send_packet(io, peer_id, CONSENSUS_DATA_PACKET, packet.clone()); - } - } - /// Called when peer sends us new private transaction packet fn on_private_transaction(&self, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { @@ -1923,16 +1677,6 @@ impl ChainSync { } Ok(()) } - - /// Broadcast private transaction message to peers. - pub fn propagate_private_transaction(&mut self, io: &mut SyncIo, packet: Bytes) { - let lucky_peers = ChainSync::select_random_peers(&self.get_private_transaction_peers()); - trace!(target: "sync", "Sending private transaction packet to {:?}", lucky_peers); - for peer_id in lucky_peers { - self.send_packet(io, peer_id, PRIVATE_TRANSACTION_PACKET, packet.clone()); - } - } - /// Called when peer sends us signed private transaction packet fn on_signed_private_transaction(&self, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { @@ -1947,32 +1691,25 @@ impl ChainSync { Ok(()) } - /// Broadcast signed private transaction message to peers. - pub fn propagate_signed_private_transaction(&mut self, io: &mut SyncIo, packet: Bytes) { - let lucky_peers = ChainSync::select_random_peers(&self.get_private_transaction_peers()); - trace!(target: "sync", "Sending signed private transaction packet to {:?}", lucky_peers); - for peer_id in lucky_peers { - self.send_packet(io, peer_id, SIGNED_PRIVATE_TRANSACTION_PACKET, packet.clone()); - } + /// propagates new transactions to all peers + pub fn propagate_new_transactions(&mut self, io: &mut SyncIo) -> usize { + SyncPropagator::propagate_new_transactions(self, io) } -} + /// Broadcast consensus message to peers. + pub fn propagate_consensus_packet(&mut self, io: &mut SyncIo, packet: Bytes) { + SyncPropagator::propagate_consensus_packet(self, io, packet); + } -/// Checks if peer is able to process service transactions -fn accepts_service_transaction(client_id: &str) -> bool { - // Parity versions starting from this will accept service-transactions - const SERVICE_TRANSACTIONS_VERSION: (u32, u32) = (1u32, 6u32); - // Parity client string prefix - const PARITY_CLIENT_ID_PREFIX: &'static str = "Parity/v"; - - if !client_id.starts_with(PARITY_CLIENT_ID_PREFIX) { - return false; - } - let ver: Vec = client_id[PARITY_CLIENT_ID_PREFIX.len()..].split('.') - .take(2) - .filter_map(|s| s.parse().ok()) - .collect(); - ver.len() == 2 && (ver[0] > SERVICE_TRANSACTIONS_VERSION.0 || (ver[0] == SERVICE_TRANSACTIONS_VERSION.0 && ver[1] >= SERVICE_TRANSACTIONS_VERSION.1)) + /// Broadcast private transaction message to peers. + pub fn propagate_private_transaction(&mut self, io: &mut SyncIo, packet: Bytes) { + SyncPropagator::propagate_private_transaction(self, io, packet); + } + + /// Broadcast signed private transaction message to peers. + pub fn propagate_signed_private_transaction(&mut self, io: &mut SyncIo, packet: Bytes) { + SyncPropagator::propagate_signed_private_transaction(self, io, packet); + } } #[cfg(test)] @@ -2144,7 +1881,7 @@ pub mod tests { let mut io = TestIo::new(&mut client, &ss, &queue, None); let peers = sync.get_lagging_peers(&chain_info); - let peer_count = sync.propagate_new_hashes(&chain_info, &mut io, &peers); + let peer_count = SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -2164,7 +1901,7 @@ pub mod tests { let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); let peers = sync.get_lagging_peers(&chain_info); - let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[], &peers); + let peer_count = SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -2185,7 +1922,7 @@ pub mod tests { let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); let peers = sync.get_lagging_peers(&chain_info); - let peer_count = sync.propagate_blocks(&chain_info, &mut io, &[hash.clone()], &peers); + let peer_count = SyncPropagator::propagate_blocks(&mut sync ,&chain_info, &mut io, &[hash.clone()], &peers); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -2224,7 +1961,7 @@ pub mod tests { }); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - sync.propagate_proposed_blocks(&mut io, &[block]); + SyncPropagator::propagate_proposed_blocks(&mut sync, &mut io, &[block]); // 1 message should be sent assert_eq!(1, io.packets.len()); @@ -2241,13 +1978,13 @@ pub mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = sync.propagate_new_transactions(&mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // Try to propagate same transactions for the second time - let peer_count2 = sync.propagate_new_transactions(&mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // Even after new block transactions should not be propagated twice sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the third time - let peer_count3 = sync.propagate_new_transactions(&mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // 1 message should be send assert_eq!(1, io.packets.len()); @@ -2268,7 +2005,7 @@ pub mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = sync.propagate_new_transactions(&mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); io.chain.insert_transaction_to_queue(); // New block import should not trigger propagation. // (we only propagate on timeout) @@ -2292,10 +2029,10 @@ pub mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = sync.propagate_new_transactions(&mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); // Try to propagate same transactions for the second time - let peer_count2 = sync.propagate_new_transactions(&mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); assert_eq!(0, io.packets.len()); assert_eq!(0, peer_count); @@ -2313,7 +2050,7 @@ pub mod tests { // should sent some { let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = sync.propagate_new_transactions(&mut io); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); assert_eq!(1, io.packets.len()); assert_eq!(1, peer_count); } @@ -2322,9 +2059,9 @@ pub mod tests { let (peer_count2, peer_count3) = { let mut io = TestIo::new(&mut client, &ss, &queue, None); // Propagate new transactions - let peer_count2 = sync.propagate_new_transactions(&mut io); + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // And now the peer should have all transactions - let peer_count3 = sync.propagate_new_transactions(&mut io); + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); (peer_count2, peer_count3) }; @@ -2347,7 +2084,7 @@ pub mod tests { let queue = RwLock::new(VecDeque::new()); let ss = TestSnapshotService::new(); let mut io = TestIo::new(&mut client, &ss, &queue, None); - sync.propagate_new_transactions(&mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); let stats = sync.transactions_stats(); assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") @@ -2377,7 +2114,7 @@ pub mod tests { io.peers_info.insert(4, "Parity/v1.7.3-ABCDEFGH".to_owned()); // and new service transaction is propagated to peers - sync.propagate_new_transactions(&mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // peer#2 && peer#4 are receiving service transaction assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET @@ -2401,7 +2138,7 @@ pub mod tests { io.peers_info.insert(1, "Parity/v1.6".to_owned()); // and service + non-service transactions are propagated to peers - sync.propagate_new_transactions(&mut io); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); // two separate packets for peer are queued: // 1) with non-service-transaction @@ -2529,7 +2266,7 @@ pub mod tests { let mut io = TestIo::new(&mut client, &ss, &queue, None); let peers = sync.get_lagging_peers(&chain_info); - sync.propagate_new_hashes(&chain_info, &mut io, &peers); + SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers); let data = &io.packets[0].data.clone(); let result = sync.on_peer_new_hashes(&mut io, 0, &Rlp::new(data)); @@ -2549,7 +2286,7 @@ pub mod tests { let mut io = TestIo::new(&mut client, &ss, &queue, None); let peers = sync.get_lagging_peers(&chain_info); - sync.propagate_blocks(&chain_info, &mut io, &[], &peers); + SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers); let data = &io.packets[0].data.clone(); let result = sync.on_peer_new_block(&mut io, 0, &Rlp::new(data)); diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs new file mode 100644 index 00000000000..f741d6bc987 --- /dev/null +++ b/ethcore/sync/src/chain/propagator.rs @@ -0,0 +1,336 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use bytes::Bytes; +use ethereum_types::{H256}; +use ethcore::client::{BlockChainInfo}; +use ethcore::header::{BlockNumber}; +use network::{PeerId, PacketId}; +use rand::Rng; +use rlp::{Encodable, RlpStream}; +use sync_io::SyncIo; +use std::cmp; +use std::collections::{HashSet}; +use transaction::SignedTransaction; + +use super::{ + random, + ChainSync, + + MAX_PEER_LAG_PROPAGATION, + MAX_PEERS_PROPAGATION, + MAX_TRANSACTION_PACKET_SIZE, + MAX_TRANSACTIONS_TO_PROPAGATE, + MIN_PEERS_PROPAGATION, + + CONSENSUS_DATA_PACKET, + NEW_BLOCK_HASHES_PACKET, + NEW_BLOCK_PACKET, + PRIVATE_TRANSACTION_PACKET, + SIGNED_PRIVATE_TRANSACTION_PACKET, + TRANSACTIONS_PACKET, +}; + +/// Checks if peer is able to process service transactions +fn accepts_service_transaction(client_id: &str) -> bool { + // Parity versions starting from this will accept service-transactions + const SERVICE_TRANSACTIONS_VERSION: (u32, u32) = (1u32, 6u32); + // Parity client string prefix + const PARITY_CLIENT_ID_PREFIX: &'static str = "Parity/v"; + + if !client_id.starts_with(PARITY_CLIENT_ID_PREFIX) { + return false; + } + let ver: Vec = client_id[PARITY_CLIENT_ID_PREFIX.len()..].split('.') + .take(2) + .filter_map(|s| s.parse().ok()) + .collect(); + ver.len() == 2 && (ver[0] > SERVICE_TRANSACTIONS_VERSION.0 || (ver[0] == SERVICE_TRANSACTIONS_VERSION.0 && ver[1] >= SERVICE_TRANSACTIONS_VERSION.1)) +} + +pub struct SyncPropagator {} + +impl SyncPropagator { + /// propagates latest block to a set of peers + pub fn propagate_blocks(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, blocks: &[H256], peers: &[PeerId]) -> usize { + trace!(target: "sync", "Sending NewBlocks to {:?}", peers); + let mut sent = 0; + for peer_id in peers { + if blocks.is_empty() { + let rlp = ChainSync::create_latest_block_rlp(io.chain()); + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); + } else { + for h in blocks { + let rlp = ChainSync::create_new_block_rlp(io.chain(), h); + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp); + } + } + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = chain_info.best_block_hash.clone(); + } + sent += 1; + } + sent + } + + /// propagates new known hashes to all peers + pub fn propagate_new_hashes(sync: &mut ChainSync, chain_info: &BlockChainInfo, io: &mut SyncIo, peers: &[PeerId]) -> usize { + trace!(target: "sync", "Sending NewHashes to {:?}", peers); + let mut sent = 0; + let last_parent = *io.chain().best_block_header().parent_hash(); + for peer_id in peers { + sent += match ChainSync::create_new_hashes_rlp(io.chain(), &last_parent, &chain_info.best_block_hash) { + Some(rlp) => { + { + if let Some(ref mut peer) = sync.peers.get_mut(peer_id) { + peer.latest_hash = chain_info.best_block_hash.clone(); + } + } + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_HASHES_PACKET, rlp); + 1 + }, + None => 0 + } + } + sent + } + + /// propagates new transactions to all peers + pub fn propagate_new_transactions(sync: &mut ChainSync, io: &mut SyncIo) -> usize { + // Early out if nobody to send to. + if sync.peers.is_empty() { + return 0; + } + + let transactions = io.chain().ready_transactions(); + if transactions.is_empty() { + return 0; + } + + let (transactions, service_transactions): (Vec<_>, Vec<_>) = transactions.iter() + .map(|tx| tx.signed()) + .partition(|tx| !tx.gas_price.is_zero()); + + // usual transactions could be propagated to all peers + let mut affected_peers = HashSet::new(); + if !transactions.is_empty() { + let peers = SyncPropagator::select_peers_for_transactions(sync, |_| true); + affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, peers, transactions); + } + + // most of times service_transactions will be empty + // => there's no need to merge packets + if !service_transactions.is_empty() { + let service_transactions_peers = SyncPropagator::select_peers_for_transactions(sync, |peer_id| accepts_service_transaction(&io.peer_info(*peer_id))); + let service_transactions_affected_peers = SyncPropagator::propagate_transactions_to_peers(sync, io, service_transactions_peers, service_transactions); + affected_peers.extend(&service_transactions_affected_peers); + } + + affected_peers.len() + } + + fn propagate_transactions_to_peers(sync: &mut ChainSync, io: &mut SyncIo, peers: Vec, transactions: Vec<&SignedTransaction>) -> HashSet { + let all_transactions_hashes = transactions.iter() + .map(|tx| tx.hash()) + .collect::>(); + let all_transactions_rlp = { + let mut packet = RlpStream::new_list(transactions.len()); + for tx in &transactions { packet.append(&**tx); } + packet.out() + }; + + // Clear old transactions from stats + sync.transactions_stats.retain(&all_transactions_hashes); + + // sqrt(x)/x scaled to max u32 + let block_number = io.chain().chain_info().best_block_number; + + let lucky_peers = { + peers.into_iter() + .filter_map(|peer_id| { + let stats = &mut sync.transactions_stats; + let peer_info = sync.peers.get_mut(&peer_id) + .expect("peer_id is form peers; peers is result of select_peers_for_transactions; select_peers_for_transactions selects peers from self.peers; qed"); + + // Send all transactions + if peer_info.last_sent_transactions.is_empty() { + // update stats + for hash in &all_transactions_hashes { + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + stats.propagated(hash, id, block_number); + } + peer_info.last_sent_transactions = all_transactions_hashes.clone(); + return Some((peer_id, all_transactions_hashes.len(), all_transactions_rlp.clone())); + } + + // Get hashes of all transactions to send to this peer + let to_send = all_transactions_hashes.difference(&peer_info.last_sent_transactions) + .take(MAX_TRANSACTIONS_TO_PROPAGATE) + .cloned() + .collect::>(); + if to_send.is_empty() { + return None; + } + + // Construct RLP + let (packet, to_send) = { + let mut to_send = to_send; + let mut packet = RlpStream::new(); + packet.begin_unbounded_list(); + let mut pushed = 0; + for tx in &transactions { + let hash = tx.hash(); + if to_send.contains(&hash) { + let mut transaction = RlpStream::new(); + tx.rlp_append(&mut transaction); + let appended = packet.append_raw_checked(&transaction.drain(), 1, MAX_TRANSACTION_PACKET_SIZE); + if !appended { + // Maximal packet size reached just proceed with sending + debug!("Transaction packet size limit reached. Sending incomplete set of {}/{} transactions.", pushed, to_send.len()); + to_send = to_send.into_iter().take(pushed).collect(); + break; + } + pushed += 1; + } + } + packet.complete_unbounded_list(); + (packet, to_send) + }; + + // Update stats + let id = io.peer_session_info(peer_id).and_then(|info| info.id); + for hash in &to_send { + // update stats + stats.propagated(hash, id, block_number); + } + + peer_info.last_sent_transactions = all_transactions_hashes + .intersection(&peer_info.last_sent_transactions) + .chain(&to_send) + .cloned() + .collect(); + Some((peer_id, to_send.len(), packet.out())) + }) + .collect::>() + }; + + // Send RLPs + let mut peers = HashSet::new(); + if lucky_peers.len() > 0 { + let mut max_sent = 0; + let lucky_peers_len = lucky_peers.len(); + for (peer_id, sent, rlp) in lucky_peers { + peers.insert(peer_id); + SyncPropagator::send_packet(io, peer_id, TRANSACTIONS_PACKET, rlp); + trace!(target: "sync", "{:02} <- Transactions ({} entries)", peer_id, sent); + max_sent = cmp::max(max_sent, sent); + } + debug!(target: "sync", "Sent up to {} transactions to {} peers.", max_sent, lucky_peers_len); + } + + peers + } + + pub fn propagate_latest_blocks(sync: &mut ChainSync, io: &mut SyncIo, sealed: &[H256]) { + let chain_info = io.chain().chain_info(); + if (((chain_info.best_block_number as i64) - (sync.last_sent_block_number as i64)).abs() as BlockNumber) < MAX_PEER_LAG_PROPAGATION { + let mut peers = sync.get_lagging_peers(&chain_info); + if sealed.is_empty() { + let hashes = SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers); + peers = ChainSync::select_random_peers(&peers); + let blocks = SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers); + if blocks != 0 || hashes != 0 { + trace!(target: "sync", "Sent latest {} blocks and {} hashes to peers.", blocks, hashes); + } + } else { + SyncPropagator::propagate_blocks(sync, &chain_info, io, sealed, &peers); + SyncPropagator::propagate_new_hashes(sync, &chain_info, io, &peers); + trace!(target: "sync", "Sent sealed block to all peers"); + }; + } + sync.last_sent_block_number = chain_info.best_block_number; + } + + /// Distribute valid proposed blocks to subset of current peers. + pub fn propagate_proposed_blocks(sync: &mut ChainSync, io: &mut SyncIo, proposed: &[Bytes]) { + let peers = sync.get_consensus_peers(); + trace!(target: "sync", "Sending proposed blocks to {:?}", peers); + for block in proposed { + let rlp = ChainSync::create_block_rlp( + block, + io.chain().chain_info().total_difficulty + ); + for peer_id in &peers { + SyncPropagator::send_packet(io, *peer_id, NEW_BLOCK_PACKET, rlp.clone()); + } + } + } + + /// Broadcast consensus message to peers. + pub fn propagate_consensus_packet(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) { + let lucky_peers = ChainSync::select_random_peers(&sync.get_consensus_peers()); + trace!(target: "sync", "Sending consensus packet to {:?}", lucky_peers); + for peer_id in lucky_peers { + SyncPropagator::send_packet(io, peer_id, CONSENSUS_DATA_PACKET, packet.clone()); + } + } + + /// Broadcast private transaction message to peers. + pub fn propagate_private_transaction(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) { + let lucky_peers = ChainSync::select_random_peers(&sync.get_private_transaction_peers()); + trace!(target: "sync", "Sending private transaction packet to {:?}", lucky_peers); + for peer_id in lucky_peers { + SyncPropagator::send_packet(io, peer_id, PRIVATE_TRANSACTION_PACKET, packet.clone()); + } + } + + /// Broadcast signed private transaction message to peers. + pub fn propagate_signed_private_transaction(sync: &mut ChainSync, io: &mut SyncIo, packet: Bytes) { + let lucky_peers = ChainSync::select_random_peers(&sync.get_private_transaction_peers()); + trace!(target: "sync", "Sending signed private transaction packet to {:?}", lucky_peers); + for peer_id in lucky_peers { + SyncPropagator::send_packet(io, peer_id, SIGNED_PRIVATE_TRANSACTION_PACKET, packet.clone()); + } + } + + fn select_peers_for_transactions(sync: &ChainSync, filter: F) -> Vec + where F: Fn(&PeerId) -> bool { + // sqrt(x)/x scaled to max u32 + let fraction = ((sync.peers.len() as f64).powf(-0.5) * (u32::max_value() as f64).round()) as u32; + let small = sync.peers.len() < MIN_PEERS_PROPAGATION; + + let mut random = random::new(); + sync.peers.keys() + .cloned() + .filter(filter) + .filter(|_| small || random.next_u32() < fraction) + .take(MAX_PEERS_PROPAGATION) + .collect() + } + + /// Generic packet sender + fn send_packet(sync: &mut SyncIo, peer_id: PeerId, packet_id: PacketId, packet: Bytes) { + if let Err(e) = sync.send(peer_id, packet_id, packet) { + debug!(target:"sync", "Error sending packet: {:?}", e); + sync.disconnect_peer(peer_id); + } + } +} + +#[cfg(test)] +mod tests { + +} From 96b9e0d3ff67167ff5cffdfe44874a5d7c26c9fa Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 16:06:01 +0200 Subject: [PATCH 06/16] Add the Chain Sync Handler --- ethcore/sync/src/chain/handler.rs | 715 ++++++++++++++++++++++++++++++ ethcore/sync/src/chain/mod.rs | 680 ++-------------------------- 2 files changed, 743 insertions(+), 652 deletions(-) create mode 100644 ethcore/sync/src/chain/handler.rs diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs new file mode 100644 index 00000000000..bb5aa5e190c --- /dev/null +++ b/ethcore/sync/src/chain/handler.rs @@ -0,0 +1,715 @@ +// Copyright 2015-2018 Parity Technologies (UK) Ltd. +// This file is part of Parity. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use api::{WARP_SYNC_PROTOCOL_ID}; +use block_sync::{BlockDownloaderImportError as DownloaderImportError, DownloadAction}; +use bytes::Bytes; +use ethcore::client::{BlockStatus, BlockId, BlockImportError, BlockImportErrorKind}; +use ethcore::error::*; +use ethcore::header::{BlockNumber, Header as BlockHeader}; +use ethcore::snapshot::{ManifestData, RestorationStatus}; +use ethereum_types::{H256, U256}; +use hash::keccak; +use network::{PeerId}; +use rlp::{Rlp}; +use snapshot::{ChunkType}; +use std::cmp; +use std::collections::{HashSet}; +use std::time::{Instant}; +use sync_io::SyncIo; + +use super::{ + BlockSet, + ChainSync, + ForkConfirmation, + PacketDecodeError, + PeerAsking, + PeerInfo, + SyncRequester, + SyncState, + + ETH_PROTOCOL_VERSION_62, + ETH_PROTOCOL_VERSION_63, + + MAX_NEW_BLOCK_AGE, + MAX_NEW_HASHES, + MAX_TRANSACTION_SIZE, + + PAR_PROTOCOL_VERSION_1, + PAR_PROTOCOL_VERSION_2, + PAR_PROTOCOL_VERSION_3, + + BLOCK_BODIES_PACKET, + BLOCK_HEADERS_PACKET, + NEW_BLOCK_HASHES_PACKET, + NEW_BLOCK_PACKET, + PRIVATE_TRANSACTION_PACKET, + RECEIPTS_PACKET, + SIGNED_PRIVATE_TRANSACTION_PACKET, + SNAPSHOT_DATA_PACKET, + SNAPSHOT_MANIFEST_PACKET, + STATUS_PACKET, + TRANSACTIONS_PACKET, +}; + +pub struct SyncHandler {} + +impl SyncHandler { + pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) { + debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); + return; + } + let rlp = Rlp::new(data); + let result = match packet_id { + STATUS_PACKET => SyncHandler::on_peer_status(sync, io, peer, &rlp), + TRANSACTIONS_PACKET => SyncHandler::on_peer_transactions(sync, io, peer, &rlp), + BLOCK_HEADERS_PACKET => SyncHandler::on_peer_block_headers(sync, io, peer, &rlp), + BLOCK_BODIES_PACKET => SyncHandler::on_peer_block_bodies(sync, io, peer, &rlp), + RECEIPTS_PACKET => SyncHandler::on_peer_block_receipts(sync, io, peer, &rlp), + NEW_BLOCK_PACKET => SyncHandler::on_peer_new_block(sync, io, peer, &rlp), + NEW_BLOCK_HASHES_PACKET => SyncHandler::on_peer_new_hashes(sync, io, peer, &rlp), + SNAPSHOT_MANIFEST_PACKET => SyncHandler::on_snapshot_manifest(sync, io, peer, &rlp), + SNAPSHOT_DATA_PACKET => SyncHandler::on_snapshot_data(sync, io, peer, &rlp), + PRIVATE_TRANSACTION_PACKET => SyncHandler::on_private_transaction(sync, io, peer, &rlp), + SIGNED_PRIVATE_TRANSACTION_PACKET => SyncHandler::on_signed_private_transaction(sync, io, peer, &rlp), + _ => { + debug!(target: "sync", "{}: Unknown packet {}", peer, packet_id); + Ok(()) + } + }; + result.unwrap_or_else(|e| { + debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e); + }) + } + + /// Called when peer sends us new consensus packet + pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + trace!(target: "sync", "Received consensus packet from {:?}", peer_id); + io.chain().queue_consensus_message(r.as_raw().to_vec()); + Ok(()) + } + + /// Called by peer when it is disconnecting + pub fn on_peer_aborting(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId) { + trace!(target: "sync", "== Disconnecting {}: {}", peer, io.peer_info(peer)); + sync.handshaking_peers.remove(&peer); + if sync.peers.contains_key(&peer) { + debug!(target: "sync", "Disconnected {}", peer); + sync.clear_peer_download(peer); + sync.peers.remove(&peer); + sync.active_peers.remove(&peer); + sync.continue_sync(io); + } + } + + /// Called when a new peer is connected + pub fn on_peer_connected(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId) { + trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer)); + if let Err(e) = sync.send_status(io, peer) { + debug!(target:"sync", "Error sending status request: {:?}", e); + io.disconnect_peer(peer); + } else { + sync.handshaking_peers.insert(peer, Instant::now()); + } + } + + /// Called by peer once it has new block bodies + pub fn on_peer_new_block(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring new block from unconfirmed peer {}", peer_id); + return Ok(()); + } + let difficulty: U256 = r.val_at(1)?; + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { + if peer.difficulty.map_or(true, |pd| difficulty > pd) { + peer.difficulty = Some(difficulty); + } + } + let block_rlp = r.at(0)?; + let header_rlp = block_rlp.at(0)?; + let h = keccak(&header_rlp.as_raw()); + trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h); + let header: BlockHeader = header_rlp.as_val()?; + if header.number() > sync.highest_block.unwrap_or(0) { + sync.highest_block = Some(header.number()); + } + let mut unknown = false; + { + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { + peer.latest_hash = header.hash(); + } + } + let last_imported_number = sync.new_blocks.last_imported_block_number(); + if last_imported_number > header.number() && last_imported_number - header.number() > MAX_NEW_BLOCK_AGE { + trace!(target: "sync", "Ignored ancient new block {:?}", h); + io.disable_peer(peer_id); + return Ok(()); + } + match io.chain().import_block(block_rlp.as_raw().to_vec()) { + Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyInChain), _)) => { + trace!(target: "sync", "New block already in chain {:?}", h); + }, + Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyQueued), _)) => { + trace!(target: "sync", "New block already queued {:?}", h); + }, + Ok(_) => { + // abort current download of the same block + sync.complete_sync(io); + sync.new_blocks.mark_as_known(&header.hash(), header.number()); + trace!(target: "sync", "New block queued {:?} ({})", h, header.number()); + }, + Err(BlockImportError(BlockImportErrorKind::Block(BlockError::UnknownParent(p)), _)) => { + unknown = true; + trace!(target: "sync", "New block with unknown parent ({:?}) {:?}", p, h); + }, + Err(e) => { + debug!(target: "sync", "Bad new block {:?} : {:?}", h, e); + io.disable_peer(peer_id); + } + }; + if unknown { + if sync.state != SyncState::Idle { + trace!(target: "sync", "NewBlock ignored while seeking"); + } else { + trace!(target: "sync", "New unknown block {:?}", h); + //TODO: handle too many unknown blocks + sync.sync_peer(io, peer_id, true); + } + } + sync.continue_sync(io); + Ok(()) + } + + /// Handles `NewHashes` packet. Initiates headers download for any unknown hashes. + pub fn on_peer_new_hashes(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring new hashes from unconfirmed peer {}", peer_id); + return Ok(()); + } + let hashes: Vec<_> = r.iter().take(MAX_NEW_HASHES).map(|item| (item.val_at::(0), item.val_at::(1))).collect(); + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { + // Peer has new blocks with unknown difficulty + peer.difficulty = None; + if let Some(&(Ok(ref h), _)) = hashes.last() { + peer.latest_hash = h.clone(); + } + } + if sync.state != SyncState::Idle { + trace!(target: "sync", "Ignoring new hashes since we're already downloading."); + let max = r.iter().take(MAX_NEW_HASHES).map(|item| item.val_at::(1).unwrap_or(0)).fold(0u64, cmp::max); + if max > sync.highest_block.unwrap_or(0) { + sync.highest_block = Some(max); + } + sync.continue_sync(io); + return Ok(()); + } + trace!(target: "sync", "{} -> NewHashes ({} entries)", peer_id, r.item_count()?); + let mut max_height: BlockNumber = 0; + let mut new_hashes = Vec::new(); + let last_imported_number = sync.new_blocks.last_imported_block_number(); + for (rh, rn) in hashes { + let hash = rh?; + let number = rn?; + if number > sync.highest_block.unwrap_or(0) { + sync.highest_block = Some(number); + } + if sync.new_blocks.is_downloading(&hash) { + continue; + } + if last_imported_number > number && last_imported_number - number > MAX_NEW_BLOCK_AGE { + trace!(target: "sync", "Ignored ancient new block hash {:?}", hash); + io.disable_peer(peer_id); + continue; + } + match io.chain().block_status(BlockId::Hash(hash.clone())) { + BlockStatus::InChain => { + trace!(target: "sync", "New block hash already in chain {:?}", hash); + }, + BlockStatus::Queued => { + trace!(target: "sync", "New hash block already queued {:?}", hash); + }, + BlockStatus::Unknown | BlockStatus::Pending => { + new_hashes.push(hash.clone()); + if number > max_height { + trace!(target: "sync", "New unknown block hash {:?}", hash); + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { + peer.latest_hash = hash.clone(); + } + max_height = number; + } + }, + BlockStatus::Bad => { + debug!(target: "sync", "Bad new block hash {:?}", hash); + io.disable_peer(peer_id); + return Ok(()); + } + } + }; + if max_height != 0 { + trace!(target: "sync", "Downloading blocks for new hashes"); + sync.new_blocks.reset_to(new_hashes); + sync.state = SyncState::NewBlocks; + sync.sync_peer(io, peer_id, true); + } + sync.continue_sync(io); + Ok(()) + } + + /// Called by peer once it has new block bodies + fn on_peer_block_bodies(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + sync.clear_peer_download(peer_id); + let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); + if !sync.reset_peer_asking(peer_id, PeerAsking::BlockBodies) { + trace!(target: "sync", "{}: Ignored unexpected bodies", peer_id); + sync.continue_sync(io); + return Ok(()); + } + let item_count = r.item_count()?; + trace!(target: "sync", "{} -> BlockBodies ({} entries), set = {:?}", peer_id, item_count, block_set); + if item_count == 0 { + sync.deactivate_peer(io, peer_id); + } + else if sync.state == SyncState::Waiting { + trace!(target: "sync", "Ignored block bodies while waiting"); + } + else + { + let result = { + let downloader = match block_set { + BlockSet::NewBlocks => &mut sync.new_blocks, + BlockSet::OldBlocks => match sync.old_blocks { + None => { + trace!(target: "sync", "Ignored block headers while block download is inactive"); + sync.continue_sync(io); + return Ok(()); + }, + Some(ref mut blocks) => blocks, + } + }; + downloader.import_bodies(io, r) + }; + + match result { + Err(DownloaderImportError::Invalid) => { + io.disable_peer(peer_id); + sync.deactivate_peer(io, peer_id); + sync.continue_sync(io); + return Ok(()); + }, + Err(DownloaderImportError::Useless) => { + sync.deactivate_peer(io, peer_id); + }, + Ok(()) => (), + } + + sync.collect_blocks(io, block_set); + sync.sync_peer(io, peer_id, false); + } + sync.continue_sync(io); + Ok(()) + } + + /// Called by peer once it has new block headers during sync + fn on_peer_block_headers(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + let confirmed = match sync.peers.get_mut(&peer_id) { + Some(ref mut peer) if peer.asking == PeerAsking::ForkHeader => { + peer.asking = PeerAsking::Nothing; + let item_count = r.item_count()?; + let (fork_number, fork_hash) = sync.fork_block.expect("ForkHeader request is sent only fork block is Some; qed").clone(); + if item_count == 0 || item_count != 1 { + trace!(target: "sync", "{}: Chain is too short to confirm the block", peer_id); + peer.confirmation = ForkConfirmation::TooShort; + } else { + let header = r.at(0)?.as_raw(); + if keccak(&header) == fork_hash { + trace!(target: "sync", "{}: Confirmed peer", peer_id); + peer.confirmation = ForkConfirmation::Confirmed; + if !io.chain_overlay().read().contains_key(&fork_number) { + io.chain_overlay().write().insert(fork_number, header.to_vec()); + } + } else { + trace!(target: "sync", "{}: Fork mismatch", peer_id); + io.disable_peer(peer_id); + return Ok(()); + } + } + true + }, + _ => false, + }; + if confirmed { + sync.sync_peer(io, peer_id, false); + return Ok(()); + } + + sync.clear_peer_download(peer_id); + let expected_hash = sync.peers.get(&peer_id).and_then(|p| p.asking_hash); + let allowed = sync.peers.get(&peer_id).map(|p| p.is_allowed()).unwrap_or(false); + let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); + if !sync.reset_peer_asking(peer_id, PeerAsking::BlockHeaders) || expected_hash.is_none() || !allowed { + trace!(target: "sync", "{}: Ignored unexpected headers, expected_hash = {:?}", peer_id, expected_hash); + sync.continue_sync(io); + return Ok(()); + } + let item_count = r.item_count()?; + trace!(target: "sync", "{} -> BlockHeaders ({} entries), state = {:?}, set = {:?}", peer_id, item_count, sync.state, block_set); + if (sync.state == SyncState::Idle || sync.state == SyncState::WaitingPeers) && sync.old_blocks.is_none() { + trace!(target: "sync", "Ignored unexpected block headers"); + sync.continue_sync(io); + return Ok(()); + } + if sync.state == SyncState::Waiting { + trace!(target: "sync", "Ignored block headers while waiting"); + sync.continue_sync(io); + return Ok(()); + } + + let result = { + let downloader = match block_set { + BlockSet::NewBlocks => &mut sync.new_blocks, + BlockSet::OldBlocks => { + match sync.old_blocks { + None => { + trace!(target: "sync", "Ignored block headers while block download is inactive"); + sync.continue_sync(io); + return Ok(()); + }, + Some(ref mut blocks) => blocks, + } + } + }; + downloader.import_headers(io, r, expected_hash) + }; + + match result { + Err(DownloaderImportError::Useless) => { + sync.deactivate_peer(io, peer_id); + }, + Err(DownloaderImportError::Invalid) => { + io.disable_peer(peer_id); + sync.deactivate_peer(io, peer_id); + sync.continue_sync(io); + return Ok(()); + }, + Ok(DownloadAction::Reset) => { + // mark all outstanding requests as expired + trace!("Resetting downloads for {:?}", block_set); + for (_, ref mut p) in sync.peers.iter_mut().filter(|&(_, ref p)| p.block_set == Some(block_set)) { + p.reset_asking(); + } + + } + Ok(DownloadAction::None) => {}, + } + + sync.collect_blocks(io, block_set); + // give a task to the same peer first if received valuable headers. + sync.sync_peer(io, peer_id, false); + // give tasks to other peers + sync.continue_sync(io); + Ok(()) + } + + /// Called by peer once it has new block receipts + fn on_peer_block_receipts(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + sync.clear_peer_download(peer_id); + let block_set = sync.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); + if !sync.reset_peer_asking(peer_id, PeerAsking::BlockReceipts) { + trace!(target: "sync", "{}: Ignored unexpected receipts", peer_id); + sync.continue_sync(io); + return Ok(()); + } + let item_count = r.item_count()?; + trace!(target: "sync", "{} -> BlockReceipts ({} entries)", peer_id, item_count); + if item_count == 0 { + sync.deactivate_peer(io, peer_id); + } + else if sync.state == SyncState::Waiting { + trace!(target: "sync", "Ignored block receipts while waiting"); + } + else + { + let result = { + let downloader = match block_set { + BlockSet::NewBlocks => &mut sync.new_blocks, + BlockSet::OldBlocks => match sync.old_blocks { + None => { + trace!(target: "sync", "Ignored block headers while block download is inactive"); + sync.continue_sync(io); + return Ok(()); + }, + Some(ref mut blocks) => blocks, + } + }; + downloader.import_receipts(io, r) + }; + + match result { + Err(DownloaderImportError::Invalid) => { + io.disable_peer(peer_id); + sync.deactivate_peer(io, peer_id); + sync.continue_sync(io); + return Ok(()); + }, + Err(DownloaderImportError::Useless) => { + sync.deactivate_peer(io, peer_id); + }, + Ok(()) => (), + } + + sync.collect_blocks(io, block_set); + sync.sync_peer(io, peer_id, false); + } + sync.continue_sync(io); + Ok(()) + } + + /// Called when snapshot manifest is downloaded from a peer. + fn on_snapshot_manifest(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id); + return Ok(()); + } + sync.clear_peer_download(peer_id); + if !sync.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || sync.state != SyncState::SnapshotManifest { + trace!(target: "sync", "{}: Ignored unexpected/expired manifest", peer_id); + sync.continue_sync(io); + return Ok(()); + } + + let manifest_rlp = r.at(0)?; + let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) { + Err(e) => { + trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e); + io.disable_peer(peer_id); + sync.continue_sync(io); + return Ok(()); + } + Ok(manifest) => manifest, + }; + + let is_supported_version = io.snapshot_service().supported_versions() + .map_or(false, |(l, h)| manifest.version >= l && manifest.version <= h); + + if !is_supported_version { + trace!(target: "sync", "{}: Snapshot manifest version not supported: {}", peer_id, manifest.version); + io.disable_peer(peer_id); + sync.continue_sync(io); + return Ok(()); + } + sync.snapshot.reset_to(&manifest, &keccak(manifest_rlp.as_raw())); + io.snapshot_service().begin_restore(manifest); + sync.state = SyncState::SnapshotData; + + // give a task to the same peer first. + sync.sync_peer(io, peer_id, false); + // give tasks to other peers + sync.continue_sync(io); + Ok(()) + } + + /// Called when snapshot data is downloaded from a peer. + fn on_snapshot_data(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id); + return Ok(()); + } + sync.clear_peer_download(peer_id); + if !sync.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || (sync.state != SyncState::SnapshotData && sync.state != SyncState::SnapshotWaiting) { + trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id); + sync.continue_sync(io); + return Ok(()); + } + + // check service status + let status = io.snapshot_service().status(); + match status { + RestorationStatus::Inactive | RestorationStatus::Failed => { + trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id); + sync.state = SyncState::WaitingPeers; + + // only note bad if restoration failed. + if let (Some(hash), RestorationStatus::Failed) = (sync.snapshot.snapshot_hash(), status) { + trace!(target: "sync", "Noting snapshot hash {} as bad", hash); + sync.snapshot.note_bad(hash); + } + + sync.snapshot.clear(); + sync.continue_sync(io); + return Ok(()); + }, + RestorationStatus::Ongoing { .. } => { + trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id); + }, + } + + let snapshot_data: Bytes = r.val_at(0)?; + match sync.snapshot.validate_chunk(&snapshot_data) { + Ok(ChunkType::Block(hash)) => { + trace!(target: "sync", "{}: Processing block chunk", peer_id); + io.snapshot_service().restore_block_chunk(hash, snapshot_data); + } + Ok(ChunkType::State(hash)) => { + trace!(target: "sync", "{}: Processing state chunk", peer_id); + io.snapshot_service().restore_state_chunk(hash, snapshot_data); + } + Err(()) => { + trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id); + io.disconnect_peer(peer_id); + sync.continue_sync(io); + return Ok(()); + } + } + + if sync.snapshot.is_complete() { + // wait for snapshot restoration process to complete + sync.state = SyncState::SnapshotWaiting; + } + // give a task to the same peer first. + sync.sync_peer(io, peer_id, false); + // give tasks to other peers + sync.continue_sync(io); + Ok(()) + } + + /// Called by peer to report status + fn on_peer_status(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + sync.handshaking_peers.remove(&peer_id); + let protocol_version: u8 = r.val_at(0)?; + let warp_protocol = io.protocol_version(&WARP_SYNC_PROTOCOL_ID, peer_id) != 0; + let peer = PeerInfo { + protocol_version: protocol_version, + network_id: r.val_at(1)?, + difficulty: Some(r.val_at(2)?), + latest_hash: r.val_at(3)?, + genesis: r.val_at(4)?, + asking: PeerAsking::Nothing, + asking_blocks: Vec::new(), + asking_hash: None, + ask_time: Instant::now(), + last_sent_transactions: HashSet::new(), + expired: false, + confirmation: if sync.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, + asking_snapshot_data: None, + snapshot_hash: if warp_protocol { Some(r.val_at(5)?) } else { None }, + snapshot_number: if warp_protocol { Some(r.val_at(6)?) } else { None }, + block_set: None, + }; + + trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{}, snapshot:{:?})", + peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis, peer.snapshot_number); + if io.is_expired() { + trace!(target: "sync", "Status packet from expired session {}:{}", peer_id, io.peer_info(peer_id)); + return Ok(()); + } + + if sync.peers.contains_key(&peer_id) { + debug!(target: "sync", "Unexpected status packet from {}:{}", peer_id, io.peer_info(peer_id)); + return Ok(()); + } + let chain_info = io.chain().chain_info(); + if peer.genesis != chain_info.genesis_hash { + io.disable_peer(peer_id); + trace!(target: "sync", "Peer {} genesis hash mismatch (ours: {}, theirs: {})", peer_id, chain_info.genesis_hash, peer.genesis); + return Ok(()); + } + if peer.network_id != sync.network_id { + io.disable_peer(peer_id); + trace!(target: "sync", "Peer {} network id mismatch (ours: {}, theirs: {})", peer_id, sync.network_id, peer.network_id); + return Ok(()); + } + if (warp_protocol && peer.protocol_version != PAR_PROTOCOL_VERSION_1 && peer.protocol_version != PAR_PROTOCOL_VERSION_2 && peer.protocol_version != PAR_PROTOCOL_VERSION_3) + || (!warp_protocol && peer.protocol_version != ETH_PROTOCOL_VERSION_63 && peer.protocol_version != ETH_PROTOCOL_VERSION_62) { + io.disable_peer(peer_id); + trace!(target: "sync", "Peer {} unsupported eth protocol ({})", peer_id, peer.protocol_version); + return Ok(()); + } + + if sync.sync_start_time.is_none() { + sync.sync_start_time = Some(Instant::now()); + } + + sync.peers.insert(peer_id.clone(), peer); + // Don't activate peer immediatelly when searching for common block. + // Let the current sync round complete first. + sync.active_peers.insert(peer_id.clone()); + debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); + if let Some((fork_block, _)) = sync.fork_block { + SyncRequester::request_fork_header_by_number(&mut sync.peers, io, peer_id, fork_block); + } else { + sync.sync_peer(io, peer_id, false); + } + Ok(()) + } + + /// Called when peer sends us new transactions + fn on_peer_transactions(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + // Accept transactions only when fully synced + if !io.is_chain_queue_empty() || (sync.state != SyncState::Idle && sync.state != SyncState::NewBlocks) { + trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id); + return Ok(()); + } + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "{} Ignoring transactions from unconfirmed/unknown peer", peer_id); + return Ok(()); + } + + let item_count = r.item_count()?; + trace!(target: "sync", "{:02} -> Transactions ({} entries)", peer_id, item_count); + let mut transactions = Vec::with_capacity(item_count); + for i in 0 .. item_count { + let rlp = r.at(i)?; + if rlp.as_raw().len() > MAX_TRANSACTION_SIZE { + debug!("Skipped oversized transaction of {} bytes", rlp.as_raw().len()); + continue; + } + let tx = rlp.as_raw().to_vec(); + transactions.push(tx); + } + io.chain().queue_transactions(transactions, peer_id); + Ok(()) + } + + /// Called when peer sends us signed private transaction packet + fn on_signed_private_transaction(sync: &ChainSync, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id); + return Ok(()); + } + + trace!(target: "sync", "Received signed private transaction packet from {:?}", peer_id); + if let Err(e) = sync.private_tx_handler.import_signed_private_transaction(r.as_raw()) { + trace!(target: "sync", "Ignoring the message, error queueing: {}", e); + } + Ok(()) + } + + /// Called when peer sends us new private transaction packet + fn on_private_transaction(sync: &ChainSync, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + if !sync.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { + trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id); + return Ok(()); + } + + trace!(target: "sync", "Received private transaction packet from {:?}", peer_id); + + if let Err(e) = sync.private_tx_handler.import_private_transaction(r.as_raw()) { + trace!(target: "sync", "Ignoring the message, error queueing: {}", e); + } + Ok(()) + } +} diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index c02759b0126..1b2f6da22b4 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -88,6 +88,7 @@ /// All other messages are ignored. /// +mod handler; mod propagator; mod requester; mod supplier; @@ -104,20 +105,20 @@ use parking_lot::RwLock; use bytes::Bytes; use rlp::{Rlp, RlpStream, DecoderError}; use network::{self, PeerId, PacketId}; -use ethcore::header::{BlockNumber, Header as BlockHeader}; -use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockImportError, BlockImportErrorKind, BlockQueueInfo}; -use ethcore::error::*; -use ethcore::snapshot::{ManifestData, RestorationStatus}; +use ethcore::header::{BlockNumber}; +use ethcore::client::{BlockChainClient, BlockStatus, BlockId, BlockChainInfo, BlockQueueInfo}; +use ethcore::snapshot::{RestorationStatus}; use sync_io::SyncIo; use super::{WarpSync, SyncConfig}; -use block_sync::{BlockDownloader, BlockDownloaderImportError as DownloaderImportError, DownloadAction}; +use block_sync::{BlockDownloader, BlockDownloaderImportError as DownloaderImportError}; use rand::Rng; -use snapshot::{Snapshot, ChunkType}; +use snapshot::{Snapshot}; use api::{EthProtocolInfo as PeerInfoDigest, WARP_SYNC_PROTOCOL_ID}; use private_tx::PrivateTxHandler; use transactions_stats::{TransactionsStats, Stats as TransactionStats}; use transaction::UnverifiedTransaction; +use self::handler::SyncHandler; use self::propagator::SyncPropagator; use self::requester::SyncRequester; use self::supplier::SyncSupplier; @@ -293,7 +294,7 @@ pub enum BlockSet { OldBlocks, } #[derive(Clone, Eq, PartialEq)] -enum ForkConfirmation { +pub enum ForkConfirmation { /// Fork block confirmation pending. Unconfirmed, /// Peers chain is too short to confirm the fork. @@ -639,559 +640,6 @@ impl ChainSync { } } - /// Called by peer to report status - fn on_peer_status(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - self.handshaking_peers.remove(&peer_id); - let protocol_version: u8 = r.val_at(0)?; - let warp_protocol = io.protocol_version(&WARP_SYNC_PROTOCOL_ID, peer_id) != 0; - let peer = PeerInfo { - protocol_version: protocol_version, - network_id: r.val_at(1)?, - difficulty: Some(r.val_at(2)?), - latest_hash: r.val_at(3)?, - genesis: r.val_at(4)?, - asking: PeerAsking::Nothing, - asking_blocks: Vec::new(), - asking_hash: None, - ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - expired: false, - confirmation: if self.fork_block.is_none() { ForkConfirmation::Confirmed } else { ForkConfirmation::Unconfirmed }, - asking_snapshot_data: None, - snapshot_hash: if warp_protocol { Some(r.val_at(5)?) } else { None }, - snapshot_number: if warp_protocol { Some(r.val_at(6)?) } else { None }, - block_set: None, - }; - - trace!(target: "sync", "New peer {} (protocol: {}, network: {:?}, difficulty: {:?}, latest:{}, genesis:{}, snapshot:{:?})", - peer_id, peer.protocol_version, peer.network_id, peer.difficulty, peer.latest_hash, peer.genesis, peer.snapshot_number); - if io.is_expired() { - trace!(target: "sync", "Status packet from expired session {}:{}", peer_id, io.peer_info(peer_id)); - return Ok(()); - } - - if self.peers.contains_key(&peer_id) { - debug!(target: "sync", "Unexpected status packet from {}:{}", peer_id, io.peer_info(peer_id)); - return Ok(()); - } - let chain_info = io.chain().chain_info(); - if peer.genesis != chain_info.genesis_hash { - io.disable_peer(peer_id); - trace!(target: "sync", "Peer {} genesis hash mismatch (ours: {}, theirs: {})", peer_id, chain_info.genesis_hash, peer.genesis); - return Ok(()); - } - if peer.network_id != self.network_id { - io.disable_peer(peer_id); - trace!(target: "sync", "Peer {} network id mismatch (ours: {}, theirs: {})", peer_id, self.network_id, peer.network_id); - return Ok(()); - } - if (warp_protocol && peer.protocol_version != PAR_PROTOCOL_VERSION_1 && peer.protocol_version != PAR_PROTOCOL_VERSION_2 && peer.protocol_version != PAR_PROTOCOL_VERSION_3) - || (!warp_protocol && peer.protocol_version != ETH_PROTOCOL_VERSION_63 && peer.protocol_version != ETH_PROTOCOL_VERSION_62) { - io.disable_peer(peer_id); - trace!(target: "sync", "Peer {} unsupported eth protocol ({})", peer_id, peer.protocol_version); - return Ok(()); - } - - if self.sync_start_time.is_none() { - self.sync_start_time = Some(Instant::now()); - } - - self.peers.insert(peer_id.clone(), peer); - // Don't activate peer immediatelly when searching for common block. - // Let the current sync round complete first. - self.active_peers.insert(peer_id.clone()); - debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); - if let Some((fork_block, _)) = self.fork_block { - SyncRequester::request_fork_header_by_number(&mut self.peers, io, peer_id, fork_block); - } else { - self.sync_peer(io, peer_id, false); - } - Ok(()) - } - - /// Called by peer once it has new block headers during sync - fn on_peer_block_headers(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - let confirmed = match self.peers.get_mut(&peer_id) { - Some(ref mut peer) if peer.asking == PeerAsking::ForkHeader => { - peer.asking = PeerAsking::Nothing; - let item_count = r.item_count()?; - let (fork_number, fork_hash) = self.fork_block.expect("ForkHeader request is sent only fork block is Some; qed").clone(); - if item_count == 0 || item_count != 1 { - trace!(target: "sync", "{}: Chain is too short to confirm the block", peer_id); - peer.confirmation = ForkConfirmation::TooShort; - } else { - let header = r.at(0)?.as_raw(); - if keccak(&header) == fork_hash { - trace!(target: "sync", "{}: Confirmed peer", peer_id); - peer.confirmation = ForkConfirmation::Confirmed; - if !io.chain_overlay().read().contains_key(&fork_number) { - io.chain_overlay().write().insert(fork_number, header.to_vec()); - } - } else { - trace!(target: "sync", "{}: Fork mismatch", peer_id); - io.disable_peer(peer_id); - return Ok(()); - } - } - true - }, - _ => false, - }; - if confirmed { - self.sync_peer(io, peer_id, false); - return Ok(()); - } - - self.clear_peer_download(peer_id); - let expected_hash = self.peers.get(&peer_id).and_then(|p| p.asking_hash); - let allowed = self.peers.get(&peer_id).map(|p| p.is_allowed()).unwrap_or(false); - let block_set = self.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); - if !self.reset_peer_asking(peer_id, PeerAsking::BlockHeaders) || expected_hash.is_none() || !allowed { - trace!(target: "sync", "{}: Ignored unexpected headers, expected_hash = {:?}", peer_id, expected_hash); - self.continue_sync(io); - return Ok(()); - } - let item_count = r.item_count()?; - trace!(target: "sync", "{} -> BlockHeaders ({} entries), state = {:?}, set = {:?}", peer_id, item_count, self.state, block_set); - if (self.state == SyncState::Idle || self.state == SyncState::WaitingPeers) && self.old_blocks.is_none() { - trace!(target: "sync", "Ignored unexpected block headers"); - self.continue_sync(io); - return Ok(()); - } - if self.state == SyncState::Waiting { - trace!(target: "sync", "Ignored block headers while waiting"); - self.continue_sync(io); - return Ok(()); - } - - let result = { - let downloader = match block_set { - BlockSet::NewBlocks => &mut self.new_blocks, - BlockSet::OldBlocks => { - match self.old_blocks { - None => { - trace!(target: "sync", "Ignored block headers while block download is inactive"); - self.continue_sync(io); - return Ok(()); - }, - Some(ref mut blocks) => blocks, - } - } - }; - downloader.import_headers(io, r, expected_hash) - }; - - match result { - Err(DownloaderImportError::Useless) => { - self.deactivate_peer(io, peer_id); - }, - Err(DownloaderImportError::Invalid) => { - io.disable_peer(peer_id); - self.deactivate_peer(io, peer_id); - self.continue_sync(io); - return Ok(()); - }, - Ok(DownloadAction::Reset) => { - // mark all outstanding requests as expired - trace!("Resetting downloads for {:?}", block_set); - for (_, ref mut p) in self.peers.iter_mut().filter(|&(_, ref p)| p.block_set == Some(block_set)) { - p.reset_asking(); - } - - } - Ok(DownloadAction::None) => {}, - } - - self.collect_blocks(io, block_set); - // give a task to the same peer first if received valuable headers. - self.sync_peer(io, peer_id, false); - // give tasks to other peers - self.continue_sync(io); - Ok(()) - } - - /// Called by peer once it has new block bodies - fn on_peer_block_bodies(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - self.clear_peer_download(peer_id); - let block_set = self.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); - if !self.reset_peer_asking(peer_id, PeerAsking::BlockBodies) { - trace!(target: "sync", "{}: Ignored unexpected bodies", peer_id); - self.continue_sync(io); - return Ok(()); - } - let item_count = r.item_count()?; - trace!(target: "sync", "{} -> BlockBodies ({} entries), set = {:?}", peer_id, item_count, block_set); - if item_count == 0 { - self.deactivate_peer(io, peer_id); - } - else if self.state == SyncState::Waiting { - trace!(target: "sync", "Ignored block bodies while waiting"); - } - else - { - let result = { - let downloader = match block_set { - BlockSet::NewBlocks => &mut self.new_blocks, - BlockSet::OldBlocks => match self.old_blocks { - None => { - trace!(target: "sync", "Ignored block headers while block download is inactive"); - self.continue_sync(io); - return Ok(()); - }, - Some(ref mut blocks) => blocks, - } - }; - downloader.import_bodies(io, r) - }; - - match result { - Err(DownloaderImportError::Invalid) => { - io.disable_peer(peer_id); - self.deactivate_peer(io, peer_id); - self.continue_sync(io); - return Ok(()); - }, - Err(DownloaderImportError::Useless) => { - self.deactivate_peer(io, peer_id); - }, - Ok(()) => (), - } - - self.collect_blocks(io, block_set); - self.sync_peer(io, peer_id, false); - } - self.continue_sync(io); - Ok(()) - } - - /// Called by peer once it has new block receipts - fn on_peer_block_receipts(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - self.clear_peer_download(peer_id); - let block_set = self.peers.get(&peer_id).and_then(|p| p.block_set).unwrap_or(BlockSet::NewBlocks); - if !self.reset_peer_asking(peer_id, PeerAsking::BlockReceipts) { - trace!(target: "sync", "{}: Ignored unexpected receipts", peer_id); - self.continue_sync(io); - return Ok(()); - } - let item_count = r.item_count()?; - trace!(target: "sync", "{} -> BlockReceipts ({} entries)", peer_id, item_count); - if item_count == 0 { - self.deactivate_peer(io, peer_id); - } - else if self.state == SyncState::Waiting { - trace!(target: "sync", "Ignored block receipts while waiting"); - } - else - { - let result = { - let downloader = match block_set { - BlockSet::NewBlocks => &mut self.new_blocks, - BlockSet::OldBlocks => match self.old_blocks { - None => { - trace!(target: "sync", "Ignored block headers while block download is inactive"); - self.continue_sync(io); - return Ok(()); - }, - Some(ref mut blocks) => blocks, - } - }; - downloader.import_receipts(io, r) - }; - - match result { - Err(DownloaderImportError::Invalid) => { - io.disable_peer(peer_id); - self.deactivate_peer(io, peer_id); - self.continue_sync(io); - return Ok(()); - }, - Err(DownloaderImportError::Useless) => { - self.deactivate_peer(io, peer_id); - }, - Ok(()) => (), - } - - self.collect_blocks(io, block_set); - self.sync_peer(io, peer_id, false); - } - self.continue_sync(io); - Ok(()) - } - - /// Called by peer once it has new block bodies - fn on_peer_new_block(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "Ignoring new block from unconfirmed peer {}", peer_id); - return Ok(()); - } - let difficulty: U256 = r.val_at(1)?; - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - if peer.difficulty.map_or(true, |pd| difficulty > pd) { - peer.difficulty = Some(difficulty); - } - } - let block_rlp = r.at(0)?; - let header_rlp = block_rlp.at(0)?; - let h = keccak(&header_rlp.as_raw()); - trace!(target: "sync", "{} -> NewBlock ({})", peer_id, h); - let header: BlockHeader = header_rlp.as_val()?; - if header.number() > self.highest_block.unwrap_or(0) { - self.highest_block = Some(header.number()); - } - let mut unknown = false; - { - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - peer.latest_hash = header.hash(); - } - } - let last_imported_number = self.new_blocks.last_imported_block_number(); - if last_imported_number > header.number() && last_imported_number - header.number() > MAX_NEW_BLOCK_AGE { - trace!(target: "sync", "Ignored ancient new block {:?}", h); - io.disable_peer(peer_id); - return Ok(()); - } - match io.chain().import_block(block_rlp.as_raw().to_vec()) { - Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyInChain), _)) => { - trace!(target: "sync", "New block already in chain {:?}", h); - }, - Err(BlockImportError(BlockImportErrorKind::Import(ImportErrorKind::AlreadyQueued), _)) => { - trace!(target: "sync", "New block already queued {:?}", h); - }, - Ok(_) => { - // abort current download of the same block - self.complete_sync(io); - self.new_blocks.mark_as_known(&header.hash(), header.number()); - trace!(target: "sync", "New block queued {:?} ({})", h, header.number()); - }, - Err(BlockImportError(BlockImportErrorKind::Block(BlockError::UnknownParent(p)), _)) => { - unknown = true; - trace!(target: "sync", "New block with unknown parent ({:?}) {:?}", p, h); - }, - Err(e) => { - debug!(target: "sync", "Bad new block {:?} : {:?}", h, e); - io.disable_peer(peer_id); - } - }; - if unknown { - if self.state != SyncState::Idle { - trace!(target: "sync", "NewBlock ignored while seeking"); - } else { - trace!(target: "sync", "New unknown block {:?}", h); - //TODO: handle too many unknown blocks - self.sync_peer(io, peer_id, true); - } - } - self.continue_sync(io); - Ok(()) - } - - /// Handles `NewHashes` packet. Initiates headers download for any unknown hashes. - fn on_peer_new_hashes(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "Ignoring new hashes from unconfirmed peer {}", peer_id); - return Ok(()); - } - let hashes: Vec<_> = r.iter().take(MAX_NEW_HASHES).map(|item| (item.val_at::(0), item.val_at::(1))).collect(); - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - // Peer has new blocks with unknown difficulty - peer.difficulty = None; - if let Some(&(Ok(ref h), _)) = hashes.last() { - peer.latest_hash = h.clone(); - } - } - if self.state != SyncState::Idle { - trace!(target: "sync", "Ignoring new hashes since we're already downloading."); - let max = r.iter().take(MAX_NEW_HASHES).map(|item| item.val_at::(1).unwrap_or(0)).fold(0u64, cmp::max); - if max > self.highest_block.unwrap_or(0) { - self.highest_block = Some(max); - } - self.continue_sync(io); - return Ok(()); - } - trace!(target: "sync", "{} -> NewHashes ({} entries)", peer_id, r.item_count()?); - let mut max_height: BlockNumber = 0; - let mut new_hashes = Vec::new(); - let last_imported_number = self.new_blocks.last_imported_block_number(); - for (rh, rn) in hashes { - let hash = rh?; - let number = rn?; - if number > self.highest_block.unwrap_or(0) { - self.highest_block = Some(number); - } - if self.new_blocks.is_downloading(&hash) { - continue; - } - if last_imported_number > number && last_imported_number - number > MAX_NEW_BLOCK_AGE { - trace!(target: "sync", "Ignored ancient new block hash {:?}", hash); - io.disable_peer(peer_id); - continue; - } - match io.chain().block_status(BlockId::Hash(hash.clone())) { - BlockStatus::InChain => { - trace!(target: "sync", "New block hash already in chain {:?}", hash); - }, - BlockStatus::Queued => { - trace!(target: "sync", "New hash block already queued {:?}", hash); - }, - BlockStatus::Unknown | BlockStatus::Pending => { - new_hashes.push(hash.clone()); - if number > max_height { - trace!(target: "sync", "New unknown block hash {:?}", hash); - if let Some(ref mut peer) = self.peers.get_mut(&peer_id) { - peer.latest_hash = hash.clone(); - } - max_height = number; - } - }, - BlockStatus::Bad => { - debug!(target: "sync", "Bad new block hash {:?}", hash); - io.disable_peer(peer_id); - return Ok(()); - } - } - }; - if max_height != 0 { - trace!(target: "sync", "Downloading blocks for new hashes"); - self.new_blocks.reset_to(new_hashes); - self.state = SyncState::NewBlocks; - self.sync_peer(io, peer_id, true); - } - self.continue_sync(io); - Ok(()) - } - - /// Called when snapshot manifest is downloaded from a peer. - fn on_snapshot_manifest(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "Ignoring snapshot manifest from unconfirmed peer {}", peer_id); - return Ok(()); - } - self.clear_peer_download(peer_id); - if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotManifest) || self.state != SyncState::SnapshotManifest { - trace!(target: "sync", "{}: Ignored unexpected/expired manifest", peer_id); - self.continue_sync(io); - return Ok(()); - } - - let manifest_rlp = r.at(0)?; - let manifest = match ManifestData::from_rlp(manifest_rlp.as_raw()) { - Err(e) => { - trace!(target: "sync", "{}: Ignored bad manifest: {:?}", peer_id, e); - io.disable_peer(peer_id); - self.continue_sync(io); - return Ok(()); - } - Ok(manifest) => manifest, - }; - - let is_supported_version = io.snapshot_service().supported_versions() - .map_or(false, |(l, h)| manifest.version >= l && manifest.version <= h); - - if !is_supported_version { - trace!(target: "sync", "{}: Snapshot manifest version not supported: {}", peer_id, manifest.version); - io.disable_peer(peer_id); - self.continue_sync(io); - return Ok(()); - } - self.snapshot.reset_to(&manifest, &keccak(manifest_rlp.as_raw())); - io.snapshot_service().begin_restore(manifest); - self.state = SyncState::SnapshotData; - - // give a task to the same peer first. - self.sync_peer(io, peer_id, false); - // give tasks to other peers - self.continue_sync(io); - Ok(()) - } - - /// Called when snapshot data is downloaded from a peer. - fn on_snapshot_data(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "Ignoring snapshot data from unconfirmed peer {}", peer_id); - return Ok(()); - } - self.clear_peer_download(peer_id); - if !self.reset_peer_asking(peer_id, PeerAsking::SnapshotData) || (self.state != SyncState::SnapshotData && self.state != SyncState::SnapshotWaiting) { - trace!(target: "sync", "{}: Ignored unexpected snapshot data", peer_id); - self.continue_sync(io); - return Ok(()); - } - - // check service status - let status = io.snapshot_service().status(); - match status { - RestorationStatus::Inactive | RestorationStatus::Failed => { - trace!(target: "sync", "{}: Snapshot restoration aborted", peer_id); - self.state = SyncState::WaitingPeers; - - // only note bad if restoration failed. - if let (Some(hash), RestorationStatus::Failed) = (self.snapshot.snapshot_hash(), status) { - trace!(target: "sync", "Noting snapshot hash {} as bad", hash); - self.snapshot.note_bad(hash); - } - - self.snapshot.clear(); - self.continue_sync(io); - return Ok(()); - }, - RestorationStatus::Ongoing { .. } => { - trace!(target: "sync", "{}: Snapshot restoration is ongoing", peer_id); - }, - } - - let snapshot_data: Bytes = r.val_at(0)?; - match self.snapshot.validate_chunk(&snapshot_data) { - Ok(ChunkType::Block(hash)) => { - trace!(target: "sync", "{}: Processing block chunk", peer_id); - io.snapshot_service().restore_block_chunk(hash, snapshot_data); - } - Ok(ChunkType::State(hash)) => { - trace!(target: "sync", "{}: Processing state chunk", peer_id); - io.snapshot_service().restore_state_chunk(hash, snapshot_data); - } - Err(()) => { - trace!(target: "sync", "{}: Got bad snapshot chunk", peer_id); - io.disconnect_peer(peer_id); - self.continue_sync(io); - return Ok(()); - } - } - - if self.snapshot.is_complete() { - // wait for snapshot restoration process to complete - self.state = SyncState::SnapshotWaiting; - } - // give a task to the same peer first. - self.sync_peer(io, peer_id, false); - // give tasks to other peers - self.continue_sync(io); - Ok(()) - } - - /// Called by peer when it is disconnecting - pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { - trace!(target: "sync", "== Disconnecting {}: {}", peer, io.peer_info(peer)); - self.handshaking_peers.remove(&peer); - if self.peers.contains_key(&peer) { - debug!(target: "sync", "Disconnected {}", peer); - self.clear_peer_download(peer); - self.peers.remove(&peer); - self.active_peers.remove(&peer); - self.continue_sync(io); - } - } - - /// Called when a new peer is connected - pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) { - trace!(target: "sync", "== Connected {}: {}", peer, io.peer_info(peer)); - if let Err(e) = self.send_status(io, peer) { - debug!(target:"sync", "Error sending status request: {:?}", e); - io.disconnect_peer(peer); - } else { - self.handshaking_peers.insert(peer, Instant::now()); - } - } - /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { let mut peers: Vec<(PeerId, U256, u8)> = self.peers.iter().filter_map(|(k, p)| @@ -1388,34 +836,6 @@ impl ChainSync { false } - /// Called when peer sends us new transactions - fn on_peer_transactions(&mut self, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - // Accept transactions only when fully synced - if !io.is_chain_queue_empty() || (self.state != SyncState::Idle && self.state != SyncState::NewBlocks) { - trace!(target: "sync", "{} Ignoring transactions while syncing", peer_id); - return Ok(()); - } - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "{} Ignoring transactions from unconfirmed/unknown peer", peer_id); - return Ok(()); - } - - let item_count = r.item_count()?; - trace!(target: "sync", "{:02} -> Transactions ({} entries)", peer_id, item_count); - let mut transactions = Vec::with_capacity(item_count); - for i in 0 .. item_count { - let rlp = r.at(i)?; - if rlp.as_raw().len() > MAX_TRANSACTION_SIZE { - debug!("Skipped oversized transaction of {} bytes", rlp.as_raw().len()); - continue; - } - let tx = rlp.as_raw().to_vec(); - transactions.push(tx); - } - io.chain().queue_transactions(transactions, peer_id); - Ok(()) - } - /// Send Status message fn send_status(&mut self, io: &mut SyncIo, peer: PeerId) -> Result<(), network::Error> { let warp_protocol_version = io.protocol_version(&WARP_SYNC_PROTOCOL_ID, peer); @@ -1447,34 +867,6 @@ impl ChainSync { SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) } - pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - if packet_id != STATUS_PACKET && !self.peers.contains_key(&peer) { - debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); - return; - } - let rlp = Rlp::new(data); - let result = match packet_id { - STATUS_PACKET => self.on_peer_status(io, peer, &rlp), - TRANSACTIONS_PACKET => self.on_peer_transactions(io, peer, &rlp), - BLOCK_HEADERS_PACKET => self.on_peer_block_headers(io, peer, &rlp), - BLOCK_BODIES_PACKET => self.on_peer_block_bodies(io, peer, &rlp), - RECEIPTS_PACKET => self.on_peer_block_receipts(io, peer, &rlp), - NEW_BLOCK_PACKET => self.on_peer_new_block(io, peer, &rlp), - NEW_BLOCK_HASHES_PACKET => self.on_peer_new_hashes(io, peer, &rlp), - SNAPSHOT_MANIFEST_PACKET => self.on_snapshot_manifest(io, peer, &rlp), - SNAPSHOT_DATA_PACKET => self.on_snapshot_data(io, peer, &rlp), - PRIVATE_TRANSACTION_PACKET => self.on_private_transaction(io, peer, &rlp), - SIGNED_PRIVATE_TRANSACTION_PACKET => self.on_signed_private_transaction(io, peer, &rlp), - _ => { - debug!(target: "sync", "{}: Unknown packet {}", peer, packet_id); - Ok(()) - } - }; - result.unwrap_or_else(|e| { - debug!(target:"sync", "{} -> Malformed packet {} : {}", peer, packet_id, e); - }) - } - pub fn maintain_peers(&mut self, io: &mut SyncIo) { let tick = Instant::now(); let mut aborting = Vec::new(); @@ -1496,7 +888,7 @@ impl ChainSync { } } for p in aborting { - self.on_peer_aborting(io, p); + SyncHandler::on_peer_aborting(self, io, p); } // Check for handshake timeouts @@ -1656,39 +1048,23 @@ impl ChainSync { } } - /// Called when peer sends us new consensus packet - fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - trace!(target: "sync", "Received consensus packet from {:?}", peer_id); - io.chain().queue_consensus_message(r.as_raw().to_vec()); - Ok(()) + pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + SyncHandler::on_packet(self, io, peer, packet_id, data); } - /// Called when peer sends us new private transaction packet - fn on_private_transaction(&self, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id); - return Ok(()); - } - - trace!(target: "sync", "Received private transaction packet from {:?}", peer_id); + /// Called when peer sends us new consensus packet + pub fn on_consensus_packet(io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + SyncHandler::on_consensus_packet(io, peer_id, r) + } - if let Err(e) = self.private_tx_handler.import_private_transaction(r.as_raw()) { - trace!(target: "sync", "Ignoring the message, error queueing: {}", e); - } - Ok(()) + /// Called by peer when it is disconnecting + pub fn on_peer_aborting(&mut self, io: &mut SyncIo, peer: PeerId) { + SyncHandler::on_peer_aborting(self, io, peer); } - /// Called when peer sends us signed private transaction packet - fn on_signed_private_transaction(&self, _io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - if !self.peers.get(&peer_id).map_or(false, |p| p.can_sync()) { - trace!(target: "sync", "{} Ignoring packet from unconfirmed/unknown peer", peer_id); - return Ok(()); - } - trace!(target: "sync", "Received signed private transaction packet from {:?}", peer_id); - if let Err(e) = self.private_tx_handler.import_signed_private_transaction(r.as_raw()) { - trace!(target: "sync", "Ignoring the message, error queueing: {}", e); - } - Ok(()) + /// Called when a new peer is connected + pub fn on_peer_connected(&mut self, io: &mut SyncIo, peer: PeerId) { + SyncHandler::on_peer_connected(self, io, peer); } /// propagates new transactions to all peers @@ -2178,7 +1554,7 @@ pub mod tests { let block = Rlp::new(&block_data); - let result = sync.on_peer_new_block(&mut io, 0, &block); + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); assert!(result.is_err()); } @@ -2197,7 +1573,7 @@ pub mod tests { let block = Rlp::new(&block_data); - let result = sync.on_peer_new_block(&mut io, 0, &block); + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); assert!(result.is_ok()); } @@ -2214,7 +1590,7 @@ pub mod tests { let empty_data = vec![]; let block = Rlp::new(&empty_data); - let result = sync.on_peer_new_block(&mut io, 0, &block); + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); assert!(result.is_err()); } @@ -2231,7 +1607,7 @@ pub mod tests { let hashes_data = get_dummy_hashes(); let hashes_rlp = Rlp::new(&hashes_data); - let result = sync.on_peer_new_hashes(&mut io, 0, &hashes_rlp); + let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); assert!(result.is_ok()); } @@ -2248,7 +1624,7 @@ pub mod tests { let empty_hashes_data = vec![]; let hashes_rlp = Rlp::new(&empty_hashes_data); - let result = sync.on_peer_new_hashes(&mut io, 0, &hashes_rlp); + let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); assert!(result.is_ok()); } @@ -2269,7 +1645,7 @@ pub mod tests { SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers); let data = &io.packets[0].data.clone(); - let result = sync.on_peer_new_hashes(&mut io, 0, &Rlp::new(data)); + let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &Rlp::new(data)); assert!(result.is_ok()); } @@ -2289,7 +1665,7 @@ pub mod tests { SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers); let data = &io.packets[0].data.clone(); - let result = sync.on_peer_new_block(&mut io, 0, &Rlp::new(data)); + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &Rlp::new(data)); assert!(result.is_ok()); } From 5a74ba9c3ecd57286a9d23df38905c12302e8b19 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 16:54:13 +0200 Subject: [PATCH 07/16] Move tests from mod -> handler --- ethcore/sync/src/chain/handler.rs | 108 ++++++++++++++++++++++++++++++ ethcore/sync/src/chain/mod.rs | 96 +------------------------- 2 files changed, 111 insertions(+), 93 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index bb5aa5e190c..0da163ddd00 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -713,3 +713,111 @@ impl SyncHandler { Ok(()) } } + +#[cfg(test)] +mod tests { + use ethcore::client::{ChainInfo, EachBlockWith, TestBlockChainClient}; + use parking_lot::RwLock; + use rlp::{Rlp}; + use std::collections::{VecDeque}; + use tests::helpers::{TestIo}; + use tests::snapshot::TestSnapshotService; + + use super::*; + use super::super::tests::{ + dummy_sync_with_peer, + get_dummy_block, + get_dummy_blocks, + get_dummy_hashes, + }; + + #[test] + fn handles_peer_new_hashes() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let hashes_data = get_dummy_hashes(); + let hashes_rlp = Rlp::new(&hashes_data); + + let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); + + assert!(result.is_ok()); + } + + #[test] + fn handles_peer_new_block_malformed() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Uncle); + + let block_data = get_dummy_block(11, client.chain_info().best_block_hash); + + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + //sync.have_common_block = true; + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let block = Rlp::new(&block_data); + + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); + + assert!(result.is_err()); + } + + #[test] + fn handles_peer_new_block() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Uncle); + + let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); + + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let block = Rlp::new(&block_data); + + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); + + assert!(result.is_ok()); + } + + #[test] + fn handles_peer_new_block_empty() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let empty_data = vec![]; + let block = Rlp::new(&empty_data); + + let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); + + assert!(result.is_err()); + } + + #[test] + fn handles_peer_new_hashes_empty() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(10, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let empty_hashes_data = vec![]; + let hashes_rlp = Rlp::new(&empty_hashes_data); + + let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); + + assert!(result.is_ok()); + } +} diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 1b2f6da22b4..6a47f07c77d 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -1107,7 +1107,7 @@ pub mod tests { use ethcore::miner::MinerService; use private_tx::NoopPrivateTxHandler; - fn get_dummy_block(order: u32, parent_hash: H256) -> Bytes { + pub fn get_dummy_block(order: u32, parent_hash: H256) -> Bytes { let mut header = Header::new(); header.set_gas_limit(0.into()); header.set_difficulty((order * 100).into()); @@ -1123,7 +1123,7 @@ pub mod tests { rlp.out() } - fn get_dummy_blocks(order: u32, parent_hash: H256) -> Bytes { + pub fn get_dummy_blocks(order: u32, parent_hash: H256) -> Bytes { let mut rlp = RlpStream::new_list(1); rlp.append_raw(&get_dummy_block(order, parent_hash), 1); let difficulty: U256 = (100 * order).into(); @@ -1131,7 +1131,7 @@ pub mod tests { rlp.out() } - fn get_dummy_hashes() -> Bytes { + pub fn get_dummy_hashes() -> Bytes { let mut rlp = RlpStream::new_list(5); for _ in 0..5 { let mut hash_d_rlp = RlpStream::new_list(2); @@ -1539,96 +1539,6 @@ pub mod tests { assert!(sent_transactions.iter().any(|tx| tx.hash() == tx2_hash)); } - #[test] - fn handles_peer_new_block_malformed() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); - - let block_data = get_dummy_block(11, client.chain_info().best_block_hash); - - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - //sync.have_common_block = true; - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let block = Rlp::new(&block_data); - - let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); - - assert!(result.is_err()); - } - - #[test] - fn handles_peer_new_block() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); - - let block_data = get_dummy_blocks(11, client.chain_info().best_block_hash); - - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let block = Rlp::new(&block_data); - - let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); - - assert!(result.is_ok()); - } - - #[test] - fn handles_peer_new_block_empty() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let empty_data = vec![]; - let block = Rlp::new(&empty_data); - - let result = SyncHandler::on_peer_new_block(&mut sync, &mut io, 0, &block); - - assert!(result.is_err()); - } - - #[test] - fn handles_peer_new_hashes() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let hashes_data = get_dummy_hashes(); - let hashes_rlp = Rlp::new(&hashes_data); - - let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); - - assert!(result.is_ok()); - } - - #[test] - fn handles_peer_new_hashes_empty() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(10, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let empty_hashes_data = vec![]; - let hashes_rlp = Rlp::new(&empty_hashes_data); - - let result = SyncHandler::on_peer_new_hashes(&mut sync, &mut io, 0, &hashes_rlp); - - assert!(result.is_ok()); - } - // idea is that what we produce when propagading latest hashes should be accepted in // on_peer_new_hashes in our code as well #[test] From ef779c737f17883ac2a5f69084f4a9c847724123 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 17:02:24 +0200 Subject: [PATCH 08/16] Move tests to propagator --- ethcore/sync/src/chain/mod.rs | 296 +------------------------- ethcore/sync/src/chain/propagator.rs | 301 +++++++++++++++++++++++++++ 2 files changed, 302 insertions(+), 295 deletions(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 6a47f07c77d..8e0c4abf823 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -1194,7 +1194,7 @@ pub mod tests { sync } - fn insert_dummy_peer(sync: &mut ChainSync, peer_id: PeerId, peer_latest_hash: H256) { + pub fn insert_dummy_peer(sync: &mut ChainSync, peer_id: PeerId, peer_latest_hash: H256) { sync.peers.insert(peer_id, PeerInfo { protocol_version: 0, @@ -1245,300 +1245,6 @@ pub mod tests { // size of three rlp encoded hash-difficulty assert_eq!(107, rlp.len()); } - - #[test] - fn sends_new_hashes_to_lagging_peer() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let chain_info = client.chain_info(); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - let peers = sync.get_lagging_peers(&chain_info); - let peer_count = SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers); - - // 1 message should be send - assert_eq!(1, io.packets.len()); - // 1 peer should be updated - assert_eq!(1, peer_count); - // NEW_BLOCK_HASHES_PACKET - assert_eq!(0x01, io.packets[0].packet_id); - } - - #[test] - fn sends_latest_block_to_lagging_peer() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let chain_info = client.chain_info(); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peers = sync.get_lagging_peers(&chain_info); - let peer_count = SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers); - - // 1 message should be send - assert_eq!(1, io.packets.len()); - // 1 peer should be updated - assert_eq!(1, peer_count); - // NEW_BLOCK_PACKET - assert_eq!(0x07, io.packets[0].packet_id); - } - - #[test] - fn sends_sealed_block() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let hash = client.block_hash(BlockId::Number(99)).unwrap(); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); - let chain_info = client.chain_info(); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peers = sync.get_lagging_peers(&chain_info); - let peer_count = SyncPropagator::propagate_blocks(&mut sync ,&chain_info, &mut io, &[hash.clone()], &peers); - - // 1 message should be send - assert_eq!(1, io.packets.len()); - // 1 peer should be updated - assert_eq!(1, peer_count); - // NEW_BLOCK_PACKET - assert_eq!(0x07, io.packets[0].packet_id); - } - - #[test] - fn sends_proposed_block() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(2, EachBlockWith::Uncle); - let queue = RwLock::new(VecDeque::new()); - let block = client.block(BlockId::Latest).unwrap().into_inner(); - let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); - sync.peers.insert(0, - PeerInfo { - // Messaging protocol - protocol_version: 2, - genesis: H256::zero(), - network_id: 0, - latest_hash: client.block_hash_delta_minus(1), - difficulty: None, - asking: PeerAsking::Nothing, - asking_blocks: Vec::new(), - asking_hash: None, - ask_time: Instant::now(), - last_sent_transactions: HashSet::new(), - expired: false, - confirmation: super::ForkConfirmation::Confirmed, - snapshot_number: None, - snapshot_hash: None, - asking_snapshot_data: None, - block_set: None, - }); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - SyncPropagator::propagate_proposed_blocks(&mut sync, &mut io, &[block]); - - // 1 message should be sent - assert_eq!(1, io.packets.len()); - // NEW_BLOCK_PACKET - assert_eq!(0x07, io.packets[0].packet_id); - } - - #[test] - fn propagates_transactions() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - client.insert_transaction_to_queue(); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - // Even after new block transactions should not be propagated twice - sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); - // Try to propagate same transactions for the third time - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - - // 1 message should be send - assert_eq!(1, io.packets.len()); - // 1 peer should be updated but only once - assert_eq!(1, peer_count); - assert_eq!(0, peer_count2); - assert_eq!(0, peer_count3); - // TRANSACTIONS_PACKET - assert_eq!(0x02, io.packets[0].packet_id); - } - - #[test] - fn does_not_propagate_new_transactions_after_new_block() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - client.insert_transaction_to_queue(); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - io.chain.insert_transaction_to_queue(); - // New block import should not trigger propagation. - // (we only propagate on timeout) - sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); - - // 2 message should be send - assert_eq!(1, io.packets.len()); - // 1 peer should receive the message - assert_eq!(1, peer_count); - // TRANSACTIONS_PACKET - assert_eq!(0x02, io.packets[0].packet_id); - } - - #[test] - fn does_not_fail_for_no_peers() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - client.insert_transaction_to_queue(); - // Sync with no peers - let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); - // Try to propagate same transactions for the second time - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - - assert_eq!(0, io.packets.len()); - assert_eq!(0, peer_count); - assert_eq!(0, peer_count2); - } - - #[test] - fn propagates_transactions_without_alternating() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - client.insert_transaction_to_queue(); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - // should sent some - { - let mut io = TestIo::new(&mut client, &ss, &queue, None); - let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - assert_eq!(1, io.packets.len()); - assert_eq!(1, peer_count); - } - // Insert some more - client.insert_transaction_to_queue(); - let (peer_count2, peer_count3) = { - let mut io = TestIo::new(&mut client, &ss, &queue, None); - // Propagate new transactions - let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - // And now the peer should have all transactions - let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - (peer_count2, peer_count3) - }; - - // 2 message should be send (in total) - assert_eq!(2, queue.read().len()); - // 1 peer should be updated but only once after inserting new transaction - assert_eq!(1, peer_count2); - assert_eq!(0, peer_count3); - // TRANSACTIONS_PACKET - assert_eq!(0x02, queue.read()[0].packet_id); - assert_eq!(0x02, queue.read()[1].packet_id); - } - - #[test] - fn should_maintain_transations_propagation_stats() { - let mut client = TestBlockChainClient::new(); - client.add_blocks(100, EachBlockWith::Uncle); - client.insert_transaction_to_queue(); - let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - - let stats = sync.transactions_stats(); - assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") - } - - #[test] - fn should_propagate_service_transaction_to_selected_peers_only() { - let mut client = TestBlockChainClient::new(); - client.insert_transaction_with_gas_price_to_queue(U256::zero()); - let block_hash = client.block_hash_delta_minus(1); - let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - // when peer#1 is Geth - insert_dummy_peer(&mut sync, 1, block_hash); - io.peers_info.insert(1, "Geth".to_owned()); - // and peer#2 is Parity, accepting service transactions - insert_dummy_peer(&mut sync, 2, block_hash); - io.peers_info.insert(2, "Parity/v1.6".to_owned()); - // and peer#3 is Parity, discarding service transactions - insert_dummy_peer(&mut sync, 3, block_hash); - io.peers_info.insert(3, "Parity/v1.5".to_owned()); - // and peer#4 is Parity, accepting service transactions - insert_dummy_peer(&mut sync, 4, block_hash); - io.peers_info.insert(4, "Parity/v1.7.3-ABCDEFGH".to_owned()); - - // and new service transaction is propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - - // peer#2 && peer#4 are receiving service transaction - assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET - assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 4)); // TRANSACTIONS_PACKET - assert_eq!(io.packets.len(), 2); - } - - #[test] - fn should_propagate_service_transaction_is_sent_as_separate_message() { - let mut client = TestBlockChainClient::new(); - let tx1_hash = client.insert_transaction_to_queue(); - let tx2_hash = client.insert_transaction_with_gas_price_to_queue(U256::zero()); - let block_hash = client.block_hash_delta_minus(1); - let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); - let queue = RwLock::new(VecDeque::new()); - let ss = TestSnapshotService::new(); - let mut io = TestIo::new(&mut client, &ss, &queue, None); - - // when peer#1 is Parity, accepting service transactions - insert_dummy_peer(&mut sync, 1, block_hash); - io.peers_info.insert(1, "Parity/v1.6".to_owned()); - - // and service + non-service transactions are propagated to peers - SyncPropagator::propagate_new_transactions(&mut sync, &mut io); - - // two separate packets for peer are queued: - // 1) with non-service-transaction - // 2) with service transaction - let sent_transactions: Vec = io.packets.iter() - .filter_map(|p| { - if p.packet_id != 0x02 || p.recipient != 1 { // TRANSACTIONS_PACKET - return None; - } - - let rlp = Rlp::new(&*p.data); - let item_count = rlp.item_count().unwrap_or(0); - if item_count != 1 { - return None; - } - - rlp.at(0).ok().and_then(|r| r.as_val().ok()) - }) - .collect(); - assert_eq!(sent_transactions.len(), 2); - assert!(sent_transactions.iter().any(|tx| tx.hash() == tx1_hash)); - assert!(sent_transactions.iter().any(|tx| tx.hash() == tx2_hash)); - } - // idea is that what we produce when propagading latest hashes should be accepted in // on_peer_new_hashes in our code as well #[test] diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index f741d6bc987..84e35e04146 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -332,5 +332,306 @@ impl SyncPropagator { #[cfg(test)] mod tests { + use ethcore::client::{BlockInfo, ChainInfo, EachBlockWith, TestBlockChainClient}; + use parking_lot::RwLock; + use private_tx::NoopPrivateTxHandler; + use rlp::{Rlp}; + use std::collections::{VecDeque}; + use tests::helpers::{TestIo}; + use tests::snapshot::TestSnapshotService; + + use super::{*, super::{*, tests::*}}; + + #[test] + fn sends_new_hashes_to_lagging_peer() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let chain_info = client.chain_info(); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + let peers = sync.get_lagging_peers(&chain_info); + let peer_count = SyncPropagator::propagate_new_hashes(&mut sync, &chain_info, &mut io, &peers); + + // 1 message should be send + assert_eq!(1, io.packets.len()); + // 1 peer should be updated + assert_eq!(1, peer_count); + // NEW_BLOCK_HASHES_PACKET + assert_eq!(0x01, io.packets[0].packet_id); + } + + #[test] + fn sends_latest_block_to_lagging_peer() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let chain_info = client.chain_info(); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peers = sync.get_lagging_peers(&chain_info); + let peer_count = SyncPropagator::propagate_blocks(&mut sync, &chain_info, &mut io, &[], &peers); + + // 1 message should be send + assert_eq!(1, io.packets.len()); + // 1 peer should be updated + assert_eq!(1, peer_count); + // NEW_BLOCK_PACKET + assert_eq!(0x07, io.packets[0].packet_id); + } + + #[test] + fn sends_sealed_block() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let hash = client.block_hash(BlockId::Number(99)).unwrap(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(5), &client); + let chain_info = client.chain_info(); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peers = sync.get_lagging_peers(&chain_info); + let peer_count = SyncPropagator::propagate_blocks(&mut sync ,&chain_info, &mut io, &[hash.clone()], &peers); + + // 1 message should be send + assert_eq!(1, io.packets.len()); + // 1 peer should be updated + assert_eq!(1, peer_count); + // NEW_BLOCK_PACKET + assert_eq!(0x07, io.packets[0].packet_id); + } + + #[test] + fn sends_proposed_block() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(2, EachBlockWith::Uncle); + let queue = RwLock::new(VecDeque::new()); + let block = client.block(BlockId::Latest).unwrap().into_inner(); + let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); + sync.peers.insert(0, + PeerInfo { + // Messaging protocol + protocol_version: 2, + genesis: H256::zero(), + network_id: 0, + latest_hash: client.block_hash_delta_minus(1), + difficulty: None, + asking: PeerAsking::Nothing, + asking_blocks: Vec::new(), + asking_hash: None, + ask_time: Instant::now(), + last_sent_transactions: HashSet::new(), + expired: false, + confirmation: ForkConfirmation::Confirmed, + snapshot_number: None, + snapshot_hash: None, + asking_snapshot_data: None, + block_set: None, + }); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + SyncPropagator::propagate_proposed_blocks(&mut sync, &mut io, &[block]); + + // 1 message should be sent + assert_eq!(1, io.packets.len()); + // NEW_BLOCK_PACKET + assert_eq!(0x07, io.packets[0].packet_id); + } + #[test] + fn propagates_transactions() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + client.insert_transaction_to_queue(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + // Try to propagate same transactions for the second time + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + // Even after new block transactions should not be propagated twice + sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); + // Try to propagate same transactions for the third time + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + + // 1 message should be send + assert_eq!(1, io.packets.len()); + // 1 peer should be updated but only once + assert_eq!(1, peer_count); + assert_eq!(0, peer_count2); + assert_eq!(0, peer_count3); + // TRANSACTIONS_PACKET + assert_eq!(0x02, io.packets[0].packet_id); + } + + #[test] + fn does_not_propagate_new_transactions_after_new_block() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + client.insert_transaction_to_queue(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + io.chain.insert_transaction_to_queue(); + // New block import should not trigger propagation. + // (we only propagate on timeout) + sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); + + // 2 message should be send + assert_eq!(1, io.packets.len()); + // 1 peer should receive the message + assert_eq!(1, peer_count); + // TRANSACTIONS_PACKET + assert_eq!(0x02, io.packets[0].packet_id); + } + + #[test] + fn does_not_fail_for_no_peers() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + client.insert_transaction_to_queue(); + // Sync with no peers + let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + sync.chain_new_blocks(&mut io, &[], &[], &[], &[], &[], &[]); + // Try to propagate same transactions for the second time + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + + assert_eq!(0, io.packets.len()); + assert_eq!(0, peer_count); + assert_eq!(0, peer_count2); + } + + #[test] + fn propagates_transactions_without_alternating() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + client.insert_transaction_to_queue(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + // should sent some + { + let mut io = TestIo::new(&mut client, &ss, &queue, None); + let peer_count = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + assert_eq!(1, io.packets.len()); + assert_eq!(1, peer_count); + } + // Insert some more + client.insert_transaction_to_queue(); + let (peer_count2, peer_count3) = { + let mut io = TestIo::new(&mut client, &ss, &queue, None); + // Propagate new transactions + let peer_count2 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + // And now the peer should have all transactions + let peer_count3 = SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + (peer_count2, peer_count3) + }; + + // 2 message should be send (in total) + assert_eq!(2, queue.read().len()); + // 1 peer should be updated but only once after inserting new transaction + assert_eq!(1, peer_count2); + assert_eq!(0, peer_count3); + // TRANSACTIONS_PACKET + assert_eq!(0x02, queue.read()[0].packet_id); + assert_eq!(0x02, queue.read()[1].packet_id); + } + + #[test] + fn should_maintain_transations_propagation_stats() { + let mut client = TestBlockChainClient::new(); + client.add_blocks(100, EachBlockWith::Uncle); + client.insert_transaction_to_queue(); + let mut sync = dummy_sync_with_peer(client.block_hash_delta_minus(1), &client); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + + let stats = sync.transactions_stats(); + assert_eq!(stats.len(), 1, "Should maintain stats for single transaction.") + } + + #[test] + fn should_propagate_service_transaction_to_selected_peers_only() { + let mut client = TestBlockChainClient::new(); + client.insert_transaction_with_gas_price_to_queue(U256::zero()); + let block_hash = client.block_hash_delta_minus(1); + let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + // when peer#1 is Geth + insert_dummy_peer(&mut sync, 1, block_hash); + io.peers_info.insert(1, "Geth".to_owned()); + // and peer#2 is Parity, accepting service transactions + insert_dummy_peer(&mut sync, 2, block_hash); + io.peers_info.insert(2, "Parity/v1.6".to_owned()); + // and peer#3 is Parity, discarding service transactions + insert_dummy_peer(&mut sync, 3, block_hash); + io.peers_info.insert(3, "Parity/v1.5".to_owned()); + // and peer#4 is Parity, accepting service transactions + insert_dummy_peer(&mut sync, 4, block_hash); + io.peers_info.insert(4, "Parity/v1.7.3-ABCDEFGH".to_owned()); + + // and new service transaction is propagated to peers + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + + // peer#2 && peer#4 are receiving service transaction + assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 2)); // TRANSACTIONS_PACKET + assert!(io.packets.iter().any(|p| p.packet_id == 0x02 && p.recipient == 4)); // TRANSACTIONS_PACKET + assert_eq!(io.packets.len(), 2); + } + + #[test] + fn should_propagate_service_transaction_is_sent_as_separate_message() { + let mut client = TestBlockChainClient::new(); + let tx1_hash = client.insert_transaction_to_queue(); + let tx2_hash = client.insert_transaction_with_gas_price_to_queue(U256::zero()); + let block_hash = client.block_hash_delta_minus(1); + let mut sync = ChainSync::new(SyncConfig::default(), &client, Arc::new(NoopPrivateTxHandler)); + let queue = RwLock::new(VecDeque::new()); + let ss = TestSnapshotService::new(); + let mut io = TestIo::new(&mut client, &ss, &queue, None); + + // when peer#1 is Parity, accepting service transactions + insert_dummy_peer(&mut sync, 1, block_hash); + io.peers_info.insert(1, "Parity/v1.6".to_owned()); + + // and service + non-service transactions are propagated to peers + SyncPropagator::propagate_new_transactions(&mut sync, &mut io); + + // two separate packets for peer are queued: + // 1) with non-service-transaction + // 2) with service transaction + let sent_transactions: Vec = io.packets.iter() + .filter_map(|p| { + if p.packet_id != 0x02 || p.recipient != 1 { // TRANSACTIONS_PACKET + return None; + } + + let rlp = Rlp::new(&*p.data); + let item_count = rlp.item_count().unwrap_or(0); + if item_count != 1 { + return None; + } + + rlp.at(0).ok().and_then(|r| r.as_val().ok()) + }) + .collect(); + assert_eq!(sent_transactions.len(), 2); + assert!(sent_transactions.iter().any(|tx| tx.hash() == tx1_hash)); + assert!(sent_transactions.iter().any(|tx| tx.hash() == tx2_hash)); + } } From e240635356939a42f482244e681ca43f24157bfc Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 17:32:28 +0200 Subject: [PATCH 09/16] Refactor SyncRequester arguments --- ethcore/sync/src/chain/handler.rs | 2 +- ethcore/sync/src/chain/mod.rs | 18 ++++----- ethcore/sync/src/chain/requester.rs | 59 ++++++++++++++--------------- 3 files changed, 39 insertions(+), 40 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index 0da163ddd00..aa69e0275c6 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -649,7 +649,7 @@ impl SyncHandler { sync.active_peers.insert(peer_id.clone()); debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); if let Some((fork_block, _)) = sync.fork_block { - SyncRequester::request_fork_header_by_number(&mut sync.peers, io, peer_id, fork_block); + SyncRequester::request_fork_header_by_number(sync, io, peer_id, fork_block); } else { sync.sync_peer(io, peer_id, false); } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 8e0c4abf823..2e0d1cf1d49 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -603,7 +603,7 @@ impl ChainSync { if !self.snapshot.have_manifest() { for p in peers { if self.peers.get(p).map_or(false, |p| p.asking == PeerAsking::Nothing) { - SyncRequester::request_snapshot_manifest(&mut self.peers, io, *p); + SyncRequester::request_snapshot_manifest(self, io, *p); } } self.state = SyncState::SnapshotManifest; @@ -732,7 +732,7 @@ impl ChainSync { // check if got new blocks to download trace!(target: "sync", "Syncing with peer {}, force={}, td={:?}, our td={}, state={:?}", peer_id, force, peer_difficulty, syncing_difficulty, self.state); if let Some(request) = self.new_blocks.request_blocks(io, num_active_peers) { - SyncRequester::request_blocks(&mut self.peers, io, peer_id, request, BlockSet::NewBlocks); + SyncRequester::request_blocks(self, io, peer_id, request, BlockSet::NewBlocks); if self.state == SyncState::Idle { self.state = SyncState::Blocks; } @@ -741,7 +741,7 @@ impl ChainSync { } if let Some(request) = self.old_blocks.as_mut().and_then(|d| d.request_blocks(io, num_active_peers)) { - SyncRequester::request_blocks(&mut self.peers, io, peer_id, request, BlockSet::OldBlocks); + SyncRequester::request_blocks(self, io, peer_id, request, BlockSet::OldBlocks); return; } }, @@ -755,7 +755,7 @@ impl ChainSync { } if peer_snapshot_hash.is_some() && peer_snapshot_hash == self.snapshot.snapshot_hash() { self.clear_peer_download(peer_id); - SyncRequester::request_snapshot_data(&mut self.peers, &mut self.snapshot, io, peer_id); + SyncRequester::request_snapshot_data(self, io, peer_id); } }, SyncState::SnapshotManifest | //already downloading from other peer @@ -862,11 +862,6 @@ impl ChainSync { io.respond(STATUS_PACKET, packet.out()) } - /// Dispatch incoming requests and responses - pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { - SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) - } - pub fn maintain_peers(&mut self, io: &mut SyncIo) { let tick = Instant::now(); let mut aborting = Vec::new(); @@ -1048,6 +1043,11 @@ impl ChainSync { } } + /// Dispatch incoming requests and responses + pub fn dispatch_packet(sync: &RwLock, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { + SyncSupplier::dispatch_packet(sync, io, peer, packet_id, data) + } + pub fn on_packet(&mut self, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { SyncHandler::on_packet(self, io, peer, packet_id, data); } diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index 506aec92fc1..594a90b88fd 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -21,14 +21,13 @@ use ethcore::header::{BlockNumber}; use ethereum_types::{H256}; use network::{PeerId, PacketId}; use rlp::{RlpStream}; -use snapshot::{Snapshot}; use std::time::{Instant}; use sync_io::SyncIo; use super::{ BlockSet, + ChainSync, PeerAsking, - Peers, ETH_PACKET_COUNT, GET_BLOCK_BODIES_PACKET, @@ -42,113 +41,113 @@ pub struct SyncRequester {} impl SyncRequester { /// Perofrm block download request` - pub fn request_blocks(peers: &mut Peers, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) { + pub fn request_blocks(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) { match request { BlockRequest::Headers { start, count, skip } => { - SyncRequester::request_headers_by_hash(peers, io, peer_id, &start, count, skip, false, block_set); + SyncRequester::request_headers_by_hash(sync, io, peer_id, &start, count, skip, false, block_set); }, BlockRequest::Bodies { hashes } => { - SyncRequester::request_bodies(peers, io, peer_id, hashes, block_set); + SyncRequester::request_bodies(sync, io, peer_id, hashes, block_set); }, BlockRequest::Receipts { hashes } => { - SyncRequester::request_receipts(peers, io, peer_id, hashes, block_set); + SyncRequester::request_receipts(sync, io, peer_id, hashes, block_set); }, } } /// Request block bodies from a peer - fn request_bodies(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { + fn request_bodies(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { let mut rlp = RlpStream::new_list(hashes.len()); trace!(target: "sync", "{} <- GetBlockBodies: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); for h in &hashes { rlp.append(&h.clone()); } - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); - let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockBodies, GET_BLOCK_BODIES_PACKET, rlp.out()); + let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); peer.asking_blocks = hashes; peer.block_set = Some(set); } /// Request headers from a peer by block number - pub fn request_fork_header_by_number(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { + pub fn request_fork_header_by_number(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { trace!(target: "sync", "{} <- GetForkHeader: at {}", peer_id, n); let mut rlp = RlpStream::new_list(4); rlp.append(&n); rlp.append(&1u32); rlp.append(&0u32); rlp.append(&0u32); - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::ForkHeader, GET_BLOCK_HEADERS_PACKET, rlp.out()); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::ForkHeader, GET_BLOCK_HEADERS_PACKET, rlp.out()); } /// Find some headers or blocks to download for a peer. - pub fn request_snapshot_data(peers: &mut Peers, snapshot: &mut Snapshot, io: &mut SyncIo, peer_id: PeerId) { + pub fn request_snapshot_data(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) { // find chunk data to download - if let Some(hash) = snapshot.needed_chunk() { - if let Some(ref mut peer) = peers.get_mut(&peer_id) { + if let Some(hash) = sync.snapshot.needed_chunk() { + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { peer.asking_snapshot_data = Some(hash.clone()); } - SyncRequester::request_snapshot_chunk(peers, io, peer_id, &hash); + SyncRequester::request_snapshot_chunk(sync, io, peer_id, &hash); } } /// Request snapshot manifest from a peer. - pub fn request_snapshot_manifest(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId) { + pub fn request_snapshot_manifest(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) { trace!(target: "sync", "{} <- GetSnapshotManifest", peer_id); let rlp = RlpStream::new_list(0); - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out()); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::SnapshotManifest, GET_SNAPSHOT_MANIFEST_PACKET, rlp.out()); } /// Request headers from a peer by block hash - fn request_headers_by_hash(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, h: &H256, count: u64, skip: u64, reverse: bool, set: BlockSet) { + fn request_headers_by_hash(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, h: &H256, count: u64, skip: u64, reverse: bool, set: BlockSet) { trace!(target: "sync", "{} <- GetBlockHeaders: {} entries starting from {}, set = {:?}", peer_id, count, h, set); let mut rlp = RlpStream::new_list(4); rlp.append(h); rlp.append(&count); rlp.append(&skip); rlp.append(&if reverse {1u32} else {0u32}); - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); - let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockHeaders, GET_BLOCK_HEADERS_PACKET, rlp.out()); + let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); peer.asking_hash = Some(h.clone()); peer.block_set = Some(set); } /// Request block receipts from a peer - fn request_receipts(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { + fn request_receipts(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, hashes: Vec, set: BlockSet) { let mut rlp = RlpStream::new_list(hashes.len()); trace!(target: "sync", "{} <- GetBlockReceipts: {} entries starting from {:?}, set = {:?}", peer_id, hashes.len(), hashes.first(), set); for h in &hashes { rlp.append(&h.clone()); } - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::BlockReceipts, GET_RECEIPTS_PACKET, rlp.out()); - let peer = peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::BlockReceipts, GET_RECEIPTS_PACKET, rlp.out()); + let peer = sync.peers.get_mut(&peer_id).expect("peer_id may originate either from on_packet, where it is already validated or from enumerating self.peers. qed"); peer.asking_blocks = hashes; peer.block_set = Some(set); } /// Request snapshot chunk from a peer. - fn request_snapshot_chunk(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, chunk: &H256) { + fn request_snapshot_chunk(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, chunk: &H256) { trace!(target: "sync", "{} <- GetSnapshotData {:?}", peer_id, chunk); let mut rlp = RlpStream::new_list(1); rlp.append(chunk); - SyncRequester::send_request(peers, sync, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out()); + SyncRequester::send_request(sync, io, peer_id, PeerAsking::SnapshotData, GET_SNAPSHOT_DATA_PACKET, rlp.out()); } /// Generic request sender - fn send_request(peers: &mut Peers, sync: &mut SyncIo, peer_id: PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { - if let Some(ref mut peer) = peers.get_mut(&peer_id) { + fn send_request(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, asking: PeerAsking, packet_id: PacketId, packet: Bytes) { + if let Some(ref mut peer) = sync.peers.get_mut(&peer_id) { if peer.asking != PeerAsking::Nothing { warn!(target:"sync", "Asking {:?} while requesting {:?}", peer.asking, asking); } peer.asking = asking; peer.ask_time = Instant::now(); let result = if packet_id >= ETH_PACKET_COUNT { - sync.send_protocol(WARP_SYNC_PROTOCOL_ID, peer_id, packet_id, packet) + io.send_protocol(WARP_SYNC_PROTOCOL_ID, peer_id, packet_id, packet) } else { - sync.send(peer_id, packet_id, packet) + io.send(peer_id, packet_id, packet) }; if let Err(e) = result { debug!(target:"sync", "Error sending request: {:?}", e); - sync.disconnect_peer(peer_id); + io.disconnect_peer(peer_id); } } } From f37c0ad38ef8363c97eab7aee53e7cc62867947d Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 18:59:20 +0200 Subject: [PATCH 10/16] Refactoring peer fork header handler --- ethcore/sync/src/chain/handler.rs | 64 ++++++++++++++++------------- ethcore/sync/src/chain/mod.rs | 2 - ethcore/sync/src/chain/requester.rs | 2 +- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index aa69e0275c6..9813b2551fe 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -323,37 +323,45 @@ impl SyncHandler { Ok(()) } + fn on_peer_fork_header(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { + { + let peer = sync.peers.get_mut(&peer_id).expect("Is only called when peer is present in peers"); + peer.asking = PeerAsking::Nothing; + let item_count = r.item_count()?; + let (fork_number, fork_hash) = sync.fork_block.expect("ForkHeader request is sent only fork block is Some; qed").clone(); + + if item_count == 0 || item_count != 1 { + trace!(target: "sync", "{}: Chain is too short to confirm the block", peer_id); + io.disable_peer(peer_id); + return Ok(()); + } + + let header = r.at(0)?.as_raw(); + if keccak(&header) != fork_hash { + trace!(target: "sync", "{}: Fork mismatch", peer_id); + io.disable_peer(peer_id); + return Ok(()); + } + + trace!(target: "sync", "{}: Confirmed peer", peer_id); + peer.confirmation = ForkConfirmation::Confirmed; + if !io.chain_overlay().read().contains_key(&fork_number) { + io.chain_overlay().write().insert(fork_number, header.to_vec()); + } + } + sync.sync_peer(io, peer_id, false); + return Ok(()); + } + /// Called by peer once it has new block headers during sync fn on_peer_block_headers(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { - let confirmed = match sync.peers.get_mut(&peer_id) { - Some(ref mut peer) if peer.asking == PeerAsking::ForkHeader => { - peer.asking = PeerAsking::Nothing; - let item_count = r.item_count()?; - let (fork_number, fork_hash) = sync.fork_block.expect("ForkHeader request is sent only fork block is Some; qed").clone(); - if item_count == 0 || item_count != 1 { - trace!(target: "sync", "{}: Chain is too short to confirm the block", peer_id); - peer.confirmation = ForkConfirmation::TooShort; - } else { - let header = r.at(0)?.as_raw(); - if keccak(&header) == fork_hash { - trace!(target: "sync", "{}: Confirmed peer", peer_id); - peer.confirmation = ForkConfirmation::Confirmed; - if !io.chain_overlay().read().contains_key(&fork_number) { - io.chain_overlay().write().insert(fork_number, header.to_vec()); - } - } else { - trace!(target: "sync", "{}: Fork mismatch", peer_id); - io.disable_peer(peer_id); - return Ok(()); - } - } - true - }, + let is_fork_header_request = match sync.peers.get(&peer_id) { + Some(peer) if peer.asking == PeerAsking::ForkHeader => true, _ => false, }; - if confirmed { - sync.sync_peer(io, peer_id, false); - return Ok(()); + + if is_fork_header_request { + return SyncHandler::on_peer_fork_header(sync, io, peer_id, r); } sync.clear_peer_download(peer_id); @@ -649,7 +657,7 @@ impl SyncHandler { sync.active_peers.insert(peer_id.clone()); debug!(target: "sync", "Connected {}:{}", peer_id, io.peer_info(peer_id)); if let Some((fork_block, _)) = sync.fork_block { - SyncRequester::request_fork_header_by_number(sync, io, peer_id, fork_block); + SyncRequester::request_fork_header(sync, io, peer_id, fork_block); } else { sync.sync_peer(io, peer_id, false); } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 2e0d1cf1d49..709225d0060 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -297,8 +297,6 @@ pub enum BlockSet { pub enum ForkConfirmation { /// Fork block confirmation pending. Unconfirmed, - /// Peers chain is too short to confirm the fork. - TooShort, /// Fork is confirmed. Confirmed, } diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index 594a90b88fd..fa52e3515f5 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -69,7 +69,7 @@ impl SyncRequester { } /// Request headers from a peer by block number - pub fn request_fork_header_by_number(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { + pub fn request_fork_header(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, n: BlockNumber) { trace!(target: "sync", "{} <- GetForkHeader: at {}", peer_id, n); let mut rlp = RlpStream::new_list(4); rlp.append(&n); From 487e51df067e8344c61a193a85063dadedaf99b1 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 19:42:53 +0200 Subject: [PATCH 11/16] Fix wrong highest block number in snapshot sync --- ethcore/sync/src/chain/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 709225d0060..c680516f660 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -548,7 +548,7 @@ impl ChainSync { // Make sure the snapshot block is not too far away from best block and network best block and // that it is higher than fork detection block let our_best_block = io.chain().chain_info().best_block_number; - let fork_block = self.fork_block.as_ref().map(|&(n, _)| n).unwrap_or(0); + let fork_block = self.fork_block.map_or(0, |(n, _)| n); let (best_hash, max_peers, snapshot_peers) = { let expected_warp_block = match self.warp_sync { @@ -558,10 +558,16 @@ impl ChainSync { //collect snapshot infos from peers let snapshots = self.peers.iter() .filter(|&(_, p)| p.is_allowed() && p.snapshot_number.map_or(false, |sn| + // Snapshot must be old enough that it's usefull to sync with it our_best_block < sn && (sn - our_best_block) > SNAPSHOT_RESTORE_THRESHOLD && + // Snapshot must have been taken after the Fork sn > fork_block && + // Snapshot must be greater than the warp barrier if any sn > expected_warp_block && - self.highest_block.map_or(true, |highest| highest >= sn && (highest - sn) <= SNAPSHOT_RESTORE_THRESHOLD) + // If we know a highest block, snapshot must be recent enough + self.highest_block.map_or(true, |highest| { + highest < sn || (highest - sn) <= SNAPSHOT_RESTORE_THRESHOLD + }) )) .filter_map(|(p, peer)| peer.snapshot_hash.map(|hash| (p, hash.clone()))) .filter(|&(_, ref hash)| !self.snapshot.is_known_bad(hash)); From 157cf3228d7be86b715edd1a1f8568b1e2d2c484 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 23 Apr 2018 21:10:29 +0200 Subject: [PATCH 12/16] Small refactor... --- ethcore/sync/src/chain/handler.rs | 8 ++++-- ethcore/sync/src/chain/mod.rs | 44 +++++++++++++++++-------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index 9813b2551fe..96636411137 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -323,6 +323,10 @@ impl SyncHandler { Ok(()) } + fn on_peer_confirmed(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId) { + sync.sync_peer(io, peer_id, false); + } + fn on_peer_fork_header(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, r: &Rlp) -> Result<(), PacketDecodeError> { { let peer = sync.peers.get_mut(&peer_id).expect("Is only called when peer is present in peers"); @@ -349,7 +353,7 @@ impl SyncHandler { io.chain_overlay().write().insert(fork_number, header.to_vec()); } } - sync.sync_peer(io, peer_id, false); + SyncHandler::on_peer_confirmed(sync, io, peer_id); return Ok(()); } @@ -659,7 +663,7 @@ impl SyncHandler { if let Some((fork_block, _)) = sync.fork_block { SyncRequester::request_fork_header(sync, io, peer_id, fork_block); } else { - sync.sync_peer(io, peer_id, false); + SyncHandler::on_peer_confirmed(sync, io, peer_id); } Ok(()) } diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index c680516f660..19b49246f76 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -416,11 +416,7 @@ impl ChainSync { pub fn new(config: SyncConfig, chain: &BlockChainClient, private_tx_handler: Arc) -> ChainSync { let chain_info = chain.chain_info(); let best_block = chain.chain_info().best_block_number; - let state = match config.warp_sync { - WarpSync::Enabled => SyncState::WaitingPeers, - WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, - _ => SyncState::Idle, - }; + let state = ChainSync::get_init_state(config.warp_sync, chain); let mut sync = ChainSync { state, @@ -445,6 +441,15 @@ impl ChainSync { sync } + fn get_init_state(warp_sync: WarpSync, chain: &BlockChainClient) -> SyncState { + let best_block = chain.chain_info().best_block_number; + match warp_sync { + WarpSync::Enabled => SyncState::WaitingPeers, + WarpSync::OnlyAndAfter(block) if block > best_block => SyncState::WaitingPeers, + _ => SyncState::Idle, + } + } + /// Returns synchonization status pub fn status(&self) -> SyncStatus { let last_imported_number = self.new_blocks.last_imported_block_number(); @@ -511,7 +516,7 @@ impl ChainSync { } } } - self.state = SyncState::Idle; + self.state = ChainSync::get_init_state(self.warp_sync, io.chain()); // Reactivate peers only if some progress has been made // since the last sync round of if starting fresh. self.active_peers = self.peers.keys().cloned().collect(); @@ -646,23 +651,24 @@ impl ChainSync { /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { - let mut peers: Vec<(PeerId, U256, u8)> = self.peers.iter().filter_map(|(k, p)| - if p.can_sync() { Some((*k, p.difficulty.unwrap_or_else(U256::zero), p.protocol_version)) } else { None }).collect(); + // Collect active peers that can sync + let mut peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| + if peer.can_sync() && self.active_peers.contains(peer_id) { + Some((*peer_id, peer.protocol_version)) + } else { + None + } + ).collect(); random::new().shuffle(&mut peers); //TODO: sort by rating // prefer peers with higher protocol version - peers.sort_by(|&(_, _, ref v1), &(_, _, ref v2)| v1.cmp(v2)); + peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); trace!(target: "sync", "Syncing with peers: {} active, {} confirmed, {} total", self.active_peers.len(), peers.len(), self.peers.len()); - for (p, _, _) in peers { - if self.active_peers.contains(&p) { - self.sync_peer(io, p, false); - } + for (peer_id, _) in peers { + self.sync_peer(io, peer_id, false); } if - self.state != SyncState::WaitingPeers && - self.state != SyncState::SnapshotWaiting && - self.state != SyncState::Waiting && - self.state != SyncState::Idle && + (self.state == SyncState::Blocks || self.state == SyncState::NewBlocks) && !self.peers.values().any(|p| p.asking != PeerAsking::Nothing && p.block_set != Some(BlockSet::OldBlocks) && p.can_sync()) { self.complete_sync(io); @@ -673,7 +679,6 @@ impl ChainSync { fn complete_sync(&mut self, io: &mut SyncIo) { trace!(target: "sync", "Sync complete"); self.reset(io); - self.state = SyncState::Idle; } /// Enter waiting state @@ -901,7 +906,7 @@ impl ChainSync { } fn check_resume(&mut self, io: &mut SyncIo) { - if self.state == SyncState::Waiting && !io.chain().queue_info().is_full() && self.state == SyncState::Waiting { + if self.state == SyncState::Waiting && !io.chain().queue_info().is_full() { self.state = SyncState::Blocks; self.continue_sync(io); } else if self.state == SyncState::SnapshotWaiting { @@ -909,7 +914,6 @@ impl ChainSync { RestorationStatus::Inactive => { trace!(target:"sync", "Snapshot restoration is complete"); self.restart(io); - self.continue_sync(io); }, RestorationStatus::Ongoing { state_chunks_done, block_chunks_done, .. } => { if !self.snapshot.is_complete() && self.snapshot.done_chunks() - (state_chunks_done + block_chunks_done) as usize <= MAX_SNAPSHOT_CHUNKS_DOWNLOAD_AHEAD { From 18af9cb2fbad9b946dec433ad51a098b75b707e3 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 7 May 2018 11:56:15 +0200 Subject: [PATCH 13/16] Address PR grumbles --- ethcore/sync/src/chain/handler.rs | 19 ++++++++----------- ethcore/sync/src/chain/mod.rs | 14 +++++++++++--- ethcore/sync/src/chain/propagator.rs | 13 ++++++------- ethcore/sync/src/chain/requester.rs | 18 +++++++++--------- ethcore/sync/src/chain/supplier.rs | 13 +++++-------- 5 files changed, 39 insertions(+), 38 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index de415f99b28..504e42d3b6d 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use api::{WARP_SYNC_PROTOCOL_ID}; +use api::WARP_SYNC_PROTOCOL_ID; use block_sync::{BlockDownloaderImportError as DownloaderImportError, DownloadAction}; use bytes::Bytes; use ethcore::client::{BlockStatus, BlockId, BlockImportError, BlockImportErrorKind}; @@ -23,12 +23,12 @@ use ethcore::header::{BlockNumber, Header as BlockHeader}; use ethcore::snapshot::{ManifestData, RestorationStatus}; use ethereum_types::{H256, U256}; use hash::keccak; -use network::{PeerId}; -use rlp::{Rlp}; -use snapshot::{ChunkType}; +use network::PeerId; +use rlp::Rlp; +use snapshot::ChunkType; use std::cmp; -use std::collections::{HashSet}; -use std::time::{Instant}; +use std::collections::HashSet; +use std::time::Instant; use sync_io::SyncIo; use super::{ @@ -40,17 +40,13 @@ use super::{ PeerInfo, SyncRequester, SyncState, - ETH_PROTOCOL_VERSION_62, ETH_PROTOCOL_VERSION_63, - MAX_NEW_BLOCK_AGE, MAX_NEW_HASHES, - PAR_PROTOCOL_VERSION_1, PAR_PROTOCOL_VERSION_2, PAR_PROTOCOL_VERSION_3, - BLOCK_BODIES_PACKET, BLOCK_HEADERS_PACKET, NEW_BLOCK_HASHES_PACKET, @@ -64,7 +60,8 @@ use super::{ TRANSACTIONS_PACKET, }; -pub struct SyncHandler {} +/// The Chain Sync Handler: handles responses from peers +pub struct SyncHandler; impl SyncHandler { pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 74e0693611c..640a9433e4c 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -651,17 +651,25 @@ impl ChainSync { /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { // Collect active peers that can sync - let mut peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| - if peer.can_sync() && self.active_peers.contains(peer_id) { + let confirmed_peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| + if peer.can_sync() { Some((*peer_id, peer.protocol_version)) } else { None } ).collect(); + let mut peers: Vec<(PeerId, u8)> = confirmed_peers.iter().filter(|&&(peer_id, _)| + self.active_peers.contains(&peer_id) + ).map(|v| *v).collect(); + random::new().shuffle(&mut peers); //TODO: sort by rating // prefer peers with higher protocol version peers.sort_by(|&(_, ref v1), &(_, ref v2)| v1.cmp(v2)); - trace!(target: "sync", "Syncing with peers: {} active, {} confirmed, {} total", self.active_peers.len(), peers.len(), self.peers.len()); + trace!( + target: "sync", + "Syncing with peers: {} active, {} confirmed, {} total", + self.active_peers.len(), confirmed_peers.len(), self.peers.len() + ); for (peer_id, _) in peers { self.sync_peer(io, peer_id, false); } diff --git a/ethcore/sync/src/chain/propagator.rs b/ethcore/sync/src/chain/propagator.rs index 84e35e04146..4ae0518a537 100644 --- a/ethcore/sync/src/chain/propagator.rs +++ b/ethcore/sync/src/chain/propagator.rs @@ -15,27 +15,25 @@ // along with Parity. If not, see . use bytes::Bytes; -use ethereum_types::{H256}; -use ethcore::client::{BlockChainInfo}; -use ethcore::header::{BlockNumber}; +use ethereum_types::H256; +use ethcore::client::BlockChainInfo; +use ethcore::header::BlockNumber; use network::{PeerId, PacketId}; use rand::Rng; use rlp::{Encodable, RlpStream}; use sync_io::SyncIo; use std::cmp; -use std::collections::{HashSet}; +use std::collections::HashSet; use transaction::SignedTransaction; use super::{ random, ChainSync, - MAX_PEER_LAG_PROPAGATION, MAX_PEERS_PROPAGATION, MAX_TRANSACTION_PACKET_SIZE, MAX_TRANSACTIONS_TO_PROPAGATE, MIN_PEERS_PROPAGATION, - CONSENSUS_DATA_PACKET, NEW_BLOCK_HASHES_PACKET, NEW_BLOCK_PACKET, @@ -61,7 +59,8 @@ fn accepts_service_transaction(client_id: &str) -> bool { ver.len() == 2 && (ver[0] > SERVICE_TRANSACTIONS_VERSION.0 || (ver[0] == SERVICE_TRANSACTIONS_VERSION.0 && ver[1] >= SERVICE_TRANSACTIONS_VERSION.1)) } -pub struct SyncPropagator {} +/// The Chain Sync Propagator: propagates data to peers +pub struct SyncPropagator; impl SyncPropagator { /// propagates latest block to a set of peers diff --git a/ethcore/sync/src/chain/requester.rs b/ethcore/sync/src/chain/requester.rs index fa52e3515f5..e6acf6bc53a 100644 --- a/ethcore/sync/src/chain/requester.rs +++ b/ethcore/sync/src/chain/requester.rs @@ -14,21 +14,20 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -use api::{WARP_SYNC_PROTOCOL_ID}; -use block_sync::{BlockRequest}; +use api::WARP_SYNC_PROTOCOL_ID; +use block_sync::BlockRequest; use bytes::Bytes; -use ethcore::header::{BlockNumber}; -use ethereum_types::{H256}; +use ethcore::header::BlockNumber; +use ethereum_types::H256; use network::{PeerId, PacketId}; -use rlp::{RlpStream}; -use std::time::{Instant}; +use rlp::RlpStream; +use std::time::Instant; use sync_io::SyncIo; use super::{ BlockSet, ChainSync, PeerAsking, - ETH_PACKET_COUNT, GET_BLOCK_BODIES_PACKET, GET_BLOCK_HEADERS_PACKET, @@ -37,10 +36,11 @@ use super::{ GET_SNAPSHOT_MANIFEST_PACKET, }; -pub struct SyncRequester {} +/// The Chain Sync Requester: requesting data to other peers +pub struct SyncRequester; impl SyncRequester { - /// Perofrm block download request` + /// Perform block download request` pub fn request_blocks(sync: &mut ChainSync, io: &mut SyncIo, peer_id: PeerId, request: BlockRequest, block_set: BlockSet) { match request { BlockRequest::Headers { start, count, skip } => { diff --git a/ethcore/sync/src/chain/supplier.rs b/ethcore/sync/src/chain/supplier.rs index 74711b51659..0bfb8569823 100644 --- a/ethcore/sync/src/chain/supplier.rs +++ b/ethcore/sync/src/chain/supplier.rs @@ -15,9 +15,9 @@ // along with Parity. If not, see . use bytes::Bytes; -use ethcore::client::{BlockId}; -use ethcore::header::{BlockNumber}; -use ethereum_types::{H256}; +use ethcore::client::BlockId; +use ethcore::header::BlockNumber; +use ethereum_types::H256; use network::{self, PeerId}; use parking_lot::RwLock; use rlp::{Rlp, RlpStream}; @@ -26,13 +26,10 @@ use sync_io::SyncIo; use super::{ ChainSync, - RlpResponseResult, PacketDecodeError, - BLOCK_BODIES_PACKET, BLOCK_HEADERS_PACKET, - CONSENSUS_DATA_PACKET, GET_BLOCK_BODIES_PACKET, GET_BLOCK_HEADERS_PACKET, @@ -40,7 +37,6 @@ use super::{ GET_RECEIPTS_PACKET, GET_SNAPSHOT_DATA_PACKET, GET_SNAPSHOT_MANIFEST_PACKET, - MAX_BODIES_TO_SEND, MAX_HEADERS_TO_SEND, MAX_NODE_DATA_TO_SEND, @@ -52,7 +48,8 @@ use super::{ SNAPSHOT_MANIFEST_PACKET, }; -pub struct SyncSupplier {} +/// The Chain Sync Supplier: answers requests from peers with available data +pub struct SyncSupplier; impl SyncSupplier { /// Dispatch incoming requests and responses From 8ad988d8080cc7e425800b8f248542caae90e07a Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 7 May 2018 14:39:16 +0200 Subject: [PATCH 14/16] Retry failed CI job --- ethcore/sync/src/chain/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 640a9433e4c..5a4c4c5a7b9 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -648,7 +648,7 @@ impl ChainSync { } } - /// Resume downloading + /// Resume downloading fn continue_sync(&mut self, io: &mut SyncIo) { // Collect active peers that can sync let confirmed_peers: Vec<(PeerId, u8)> = self.peers.iter().filter_map(|(peer_id, peer)| From 14979ffb538a17e290a0b3bebacfbc9cde5c5194 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Mon, 7 May 2018 18:09:14 +0200 Subject: [PATCH 15/16] Fix tests --- ethcore/sync/src/tests/snapshot.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ethcore/sync/src/tests/snapshot.rs b/ethcore/sync/src/tests/snapshot.rs index 2f6441f4f28..804ebe9c535 100644 --- a/ethcore/sync/src/tests/snapshot.rs +++ b/ethcore/sync/src/tests/snapshot.rs @@ -22,7 +22,7 @@ use parking_lot::Mutex; use bytes::Bytes; use ethcore::snapshot::{SnapshotService, ManifestData, RestorationStatus}; use ethcore::header::BlockNumber; -use ethcore::client::{EachBlockWith}; +use ethcore::client::EachBlockWith; use super::helpers::*; use {SyncConfig, WarpSync}; @@ -99,7 +99,15 @@ impl SnapshotService for TestSnapshotService { } fn begin_restore(&self, manifest: ManifestData) { - *self.restoration_manifest.lock() = Some(manifest); + let mut restoration_manifest = self.restoration_manifest.lock(); + + if let Some(ref c_manifest) = *restoration_manifest { + if c_manifest.state_root == manifest.state_root { + return; + } + } + + *restoration_manifest = Some(manifest); self.state_restoration_chunks.lock().clear(); self.block_restoration_chunks.lock().clear(); } From 79a46f9ef4432616cdae907c79ddbd5a01bf28a6 Mon Sep 17 00:00:00 2001 From: Nicolas Gotchac Date: Tue, 8 May 2018 19:06:19 +0200 Subject: [PATCH 16/16] PR Grumbles --- ethcore/sync/src/chain/handler.rs | 1 + ethcore/sync/src/chain/mod.rs | 145 +++++++++++++++--------------- 2 files changed, 73 insertions(+), 73 deletions(-) diff --git a/ethcore/sync/src/chain/handler.rs b/ethcore/sync/src/chain/handler.rs index 504e42d3b6d..966b7ce20ad 100644 --- a/ethcore/sync/src/chain/handler.rs +++ b/ethcore/sync/src/chain/handler.rs @@ -64,6 +64,7 @@ use super::{ pub struct SyncHandler; impl SyncHandler { + /// Handle incoming packet from peer pub fn on_packet(sync: &mut ChainSync, io: &mut SyncIo, peer: PeerId, packet_id: u8, data: &[u8]) { if packet_id != STATUS_PACKET && !sync.peers.contains_key(&peer) { debug!(target:"sync", "Unexpected packet {} from unregistered peer: {}:{}", packet_id, peer, io.peer_info(peer)); diff --git a/ethcore/sync/src/chain/mod.rs b/ethcore/sync/src/chain/mod.rs index 5a4c4c5a7b9..e76cfda14dc 100644 --- a/ethcore/sync/src/chain/mod.rs +++ b/ethcore/sync/src/chain/mod.rs @@ -14,79 +14,78 @@ // You should have received a copy of the GNU General Public License // along with Parity. If not, see . -/// `BlockChain` synchronization strategy. -/// Syncs to peers and keeps up to date. -/// This implementation uses ethereum protocol v63 -/// -/// Syncing strategy summary. -/// Split the chain into ranges of N blocks each. Download ranges sequentially. Split each range into subchains of M blocks. Download subchains in parallel. -/// State. -/// Sync state consists of the following data: -/// - s: State enum which can be one of the following values: `ChainHead`, `Blocks`, `Idle` -/// - H: A set of downloaded block headers -/// - B: A set of downloaded block bodies -/// - S: Set of block subchain start block hashes to download. -/// - l: Last imported / common block hash -/// - P: A set of connected peers. For each peer we maintain its last known total difficulty and starting block hash being requested if any. -/// General behaviour. -/// We start with all sets empty, l is set to the best block in the block chain, s is set to `ChainHead`. -/// If at any moment a bad block is reported by the block queue, we set s to `ChainHead`, reset l to the best block in the block chain and clear H, B and S. -/// If at any moment P becomes empty, we set s to `ChainHead`, and clear H, B and S. -/// -/// Workflow for `ChainHead` state. -/// In this state we try to get subchain headers with a single `GetBlockHeaders` request. -/// On `NewPeer` / On `Restart`: -/// If peer's total difficulty is higher and there are less than 5 peers downloading, request N/M headers with interval M+1 starting from l -/// On `BlockHeaders(R)`: -/// If R is empty: -/// If l is equal to genesis block hash or l is more than 1000 blocks behind our best hash: -/// Remove current peer from P. set l to the best block in the block chain. Select peer with maximum total difficulty from P and restart. -/// Else -/// Set l to l’s parent and restart. -/// Else if we already have all the headers in the block chain or the block queue: -/// Set s to `Idle`, -/// Else -/// Set S to R, set s to `Blocks`. -/// -/// All other messages are ignored. -/// -/// Workflow for `Blocks` state. -/// In this state we download block headers and bodies from multiple peers. -/// On `NewPeer` / On `Restart`: -/// For all idle peers: -/// Find a set of 256 or less block hashes in H which are not in B and not being downloaded by other peers. If the set is not empty: -/// Request block bodies for the hashes in the set. -/// Else -/// Find an element in S which is not being downloaded by other peers. If found: Request M headers starting from the element. -/// -/// On `BlockHeaders(R)`: -/// If R is empty remove current peer from P and restart. -/// Validate received headers: -/// For each header find a parent in H or R or the blockchain. Restart if there is a block with unknown parent. -/// Find at least one header from the received list in S. Restart if there is none. -/// Go to `CollectBlocks`. -/// -/// On `BlockBodies(R)`: -/// If R is empty remove current peer from P and restart. -/// Add bodies with a matching header in H to B. -/// Go to `CollectBlocks`. -/// -/// `CollectBlocks`: -/// Find a chain of blocks C in H starting from h where h’s parent equals to l. The chain ends with the first block which does not have a body in B. -/// Add all blocks from the chain to the block queue. Remove them from H and B. Set l to the hash of the last block from C. -/// Update and merge subchain heads in S. For each h in S find a chain of blocks in B starting from h. Remove h from S. if the chain does not include an element from S add the end of the chain to S. -/// If H is empty and S contains a single element set s to `ChainHead`. -/// Restart. -/// -/// All other messages are ignored. -/// Workflow for Idle state. -/// On `NewBlock`: -/// Import the block. If the block is unknown set s to `ChainHead` and restart. -/// On `NewHashes`: -/// Set s to `ChainHead` and restart. -/// -/// All other messages are ignored. -/// +//! `BlockChain` synchronization strategy. +//! Syncs to peers and keeps up to date. +//! This implementation uses ethereum protocol v63 +//! +//! Syncing strategy summary. +//! Split the chain into ranges of N blocks each. Download ranges sequentially. Split each range into subchains of M blocks. Download subchains in parallel. +//! State. +//! Sync state consists of the following data: +//! - s: State enum which can be one of the following values: `ChainHead`, `Blocks`, `Idle` +//! - H: A set of downloaded block headers +//! - B: A set of downloaded block bodies +//! - S: Set of block subchain start block hashes to download. +//! - l: Last imported / common block hash +//! - P: A set of connected peers. For each peer we maintain its last known total difficulty and starting block hash being requested if any. +//! General behaviour. +//! We start with all sets empty, l is set to the best block in the block chain, s is set to `ChainHead`. +//! If at any moment a bad block is reported by the block queue, we set s to `ChainHead`, reset l to the best block in the block chain and clear H, B and S. +//! If at any moment P becomes empty, we set s to `ChainHead`, and clear H, B and S. +//! +//! Workflow for `ChainHead` state. +//! In this state we try to get subchain headers with a single `GetBlockHeaders` request. +//! On `NewPeer` / On `Restart`: +//! If peer's total difficulty is higher and there are less than 5 peers downloading, request N/M headers with interval M+1 starting from l +//! On `BlockHeaders(R)`: +//! If R is empty: +//! If l is equal to genesis block hash or l is more than 1000 blocks behind our best hash: +//! Remove current peer from P. set l to the best block in the block chain. Select peer with maximum total difficulty from P and restart. +//! Else +//! Set l to l’s parent and restart. +//! Else if we already have all the headers in the block chain or the block queue: +//! Set s to `Idle`, +//! Else +//! Set S to R, set s to `Blocks`. +//! +//! All other messages are ignored. +//! +//! Workflow for `Blocks` state. +//! In this state we download block headers and bodies from multiple peers. +//! On `NewPeer` / On `Restart`: +//! For all idle peers: +//! Find a set of 256 or less block hashes in H which are not in B and not being downloaded by other peers. If the set is not empty: +//! Request block bodies for the hashes in the set. +//! Else +//! Find an element in S which is not being downloaded by other peers. If found: Request M headers starting from the element. +//! +//! On `BlockHeaders(R)`: +//! If R is empty remove current peer from P and restart. +//! Validate received headers: +//! For each header find a parent in H or R or the blockchain. Restart if there is a block with unknown parent. +//! Find at least one header from the received list in S. Restart if there is none. +//! Go to `CollectBlocks`. +//! +//! On `BlockBodies(R)`: +//! If R is empty remove current peer from P and restart. +//! Add bodies with a matching header in H to B. +//! Go to `CollectBlocks`. +//! +//! `CollectBlocks`: +//! Find a chain of blocks C in H starting from h where h’s parent equals to l. The chain ends with the first block which does not have a body in B. +//! Add all blocks from the chain to the block queue. Remove them from H and B. Set l to the hash of the last block from C. +//! Update and merge subchain heads in S. For each h in S find a chain of blocks in B starting from h. Remove h from S. if the chain does not include an element from S add the end of the chain to S. +//! If H is empty and S contains a single element set s to `ChainHead`. +//! Restart. +//! +//! All other messages are ignored. +//! Workflow for Idle state. +//! On `NewBlock`: +//! Import the block. If the block is unknown set s to `ChainHead` and restart. +//! On `NewHashes`: +//! Set s to `ChainHead` and restart. +//! +//! All other messages are ignored. mod handler; mod propagator;