diff --git a/Cargo.lock b/Cargo.lock index 32447a6892f91..0396556fa5105 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -168,14 +168,14 @@ dependencies = [ [[package]] name = "clap" -version = "2.31.2" +version = "2.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "yaml-rust 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -301,7 +301,7 @@ dependencies = [ name = "demo-cli" version = "0.1.0" dependencies = [ - "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)", "demo-executor 0.1.0", "demo-primitives 0.1.0", @@ -1401,7 +1401,7 @@ dependencies = [ "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)", - "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "ctrlc 1.1.1 (git+https://github.com/paritytech/rust-ctrlc.git)", "ed25519 0.1.0", "env_logger 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1420,6 +1420,7 @@ dependencies = [ "serde_json 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", "slog 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-client 0.1.0", + "substrate-codec 0.1.0", "substrate-network 0.1.0", "substrate-primitives 0.1.0", "substrate-rpc 0.1.0", @@ -1538,7 +1539,7 @@ dependencies = [ name = "polkadot-service" version = "0.2.0" dependencies = [ - "clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "ed25519 0.1.0", "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", "exit-future 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2687,7 +2688,7 @@ dependencies = [ [[package]] name = "textwrap" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "unicode-width 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3249,7 +3250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cc 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "49ec142f5768efb5b7622aebc3fdbdbb8950a4b9ba996393cb76ef7466e8747d" "checksum cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "405216fd8fe65f718daa7102ea808a946b6ce40c742998fbfd3463645552de18" "checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6" -"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536" +"checksum clap 2.32.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b957d88f4b6a63b9d70d5f454ac8011819c6efa7727858f458ab71c756ce2d3e" "checksum cmake 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "95470235c31c726d72bf2e1f421adc1e65b9d561bf5529612cbe1a72da1467b3" "checksum constant_time_eq 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8ff012e225ce166d4422e0e78419d901719760f62ae2b7969ca6b564d1b54a9e" "checksum core-foundation 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "25bfd746d203017f7d5cbd31ee5d8e17f94b6521c7af77ece6c9e4b2d4b16c67" @@ -3431,7 +3432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum take_mut 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" -"checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" +"checksum textwrap 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "307686869c93e71f94da64286f9a9524c0f308a9e1c87a583de8e9c9039ad3f6" "checksum thread_local 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "279ef31c19ededf577bfd12dfae728040a21f635b06a24cd670ff510edd38963" "checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" "checksum timer 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "31d42176308937165701f50638db1c31586f183f1aab416268216577aec7306b" diff --git a/polkadot/cli/Cargo.toml b/polkadot/cli/Cargo.toml index f642ca27dc19d..76a899de7cd04 100644 --- a/polkadot/cli/Cargo.toml +++ b/polkadot/cli/Cargo.toml @@ -26,6 +26,7 @@ parking_lot = "0.4" serde_json = "1.0" serde = "1.0" substrate-client = { path = "../../substrate/client" } +substrate-codec = { path = "../../substrate/codec" } substrate-network = { path = "../../substrate/network" } substrate-primitives = { path = "../../substrate/primitives" } substrate-rpc = { path = "../../substrate/rpc" } diff --git a/polkadot/cli/src/cli.yml b/polkadot/cli/src/cli.yml index 7af6eeb917801..9a5033831150c 100644 --- a/polkadot/cli/src/cli.yml +++ b/polkadot/cli/src/cli.yml @@ -104,3 +104,53 @@ subcommands: value_name: CHAIN_SPEC help: Specify the chain specification (one of dev, local or poc-2) takes_value: true + - export-blocks: + about: Export blocks to a file + args: + - OUTPUT: + index: 1 + help: Output file name or stdout if unspecified. + required: false + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true + - from: + long: from + value_name: BLOCK + help: Specify starting block number. 1 by default. + takes_value: true + - to: + long: to + value_name: BLOCK + help: Specify last block number. Best block by default. + takes_value: true + - json: + long: json + help: Use JSON output rather than binary. + takes_value: false + - import-blocks: + about: Import blocks from file. + args: + - INPUT: + index: 1 + help: Input file or stdin if unspecified. + required: false + - chain: + long: chain + value_name: CHAIN_SPEC + help: Specify the chain specification. + takes_value: true + - base-path: + long: base-path + short: d + value_name: PATH + help: Specify custom base path. + takes_value: true diff --git a/polkadot/cli/src/lib.rs b/polkadot/cli/src/lib.rs index 21399e0ed750d..f96690f79eaa6 100644 --- a/polkadot/cli/src/lib.rs +++ b/polkadot/cli/src/lib.rs @@ -36,6 +36,7 @@ extern crate serde_json; extern crate substrate_client as client; extern crate substrate_network as network; +extern crate substrate_codec as codec; extern crate substrate_primitives; extern crate substrate_rpc; extern crate substrate_rpc_servers as rpc; @@ -65,11 +66,15 @@ mod chain_spec; pub use chain_spec::ChainSpec; -use std::io; +use std::io::{self, Write, Read, stdin, stdout}; +use std::fs::File; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use substrate_telemetry::{init_telemetry, TelemetryConfig}; -use polkadot_primitives::Block; +use polkadot_primitives::{Block, BlockId}; +use codec::Slicable; +use client::BlockOrigin; +use runtime_primitives::generic::SignedBlock; use futures::sync::mpsc; use futures::{Sink, Future, Stream}; @@ -106,6 +111,12 @@ fn load_spec(matches: &clap::ArgMatches) -> Result { Ok(spec) } +fn base_path(matches: &clap::ArgMatches) -> PathBuf { + matches.value_of("base-path") + .map(|x| Path::new(x).to_owned()) + .unwrap_or_else(default_base_path) +} + /// Parse command line arguments and start the node. /// /// IANA unassigned port ranges that we could use: @@ -123,10 +134,10 @@ pub fn run(args: I) -> error::Result<()> where Ok(m) => m, Err(ref e) if e.kind == clap::ErrorKind::VersionDisplayed => return Ok(()), Err(ref e) if e.kind == clap::ErrorKind::HelpDisplayed => { - let _ = clap::App::from_yaml(yaml).print_long_help(); + print!("{}", e); return Ok(()) } - Err(e) => return Err(e.into()), + Err(e) => e.exit(), }; // TODO [ToDr] Split parameters parsing from actual execution. @@ -139,11 +150,15 @@ pub fn run(args: I) -> error::Result<()> where info!(" by Parity Technologies, 2017, 2018"); if let Some(matches) = matches.subcommand_matches("build-spec") { - let spec = load_spec(&matches)?; - info!("Building chain spec"); - let json = spec.to_json(matches.is_present("raw"))?; - print!("{}", json); - return Ok(()) + return build_spec(matches); + } + + if let Some(matches) = matches.subcommand_matches("export-blocks") { + return export_blocks(matches); + } + + if let Some(matches) = matches.subcommand_matches("import-blocks") { + return import_blocks(matches); } let spec = load_spec(&matches)?; @@ -154,10 +169,7 @@ pub fn run(args: I) -> error::Result<()> where info!("Node name: {}", config.name); } - let base_path = matches.value_of("base-path") - .map(|x| Path::new(x).to_owned()) - .unwrap_or_else(default_base_path); - + let base_path = base_path(&matches); config.keystore_path = matches.value_of("keystore") .map(|x| Path::new(x).to_owned()) .unwrap_or_else(|| keystore_path(&base_path)) @@ -245,6 +257,119 @@ pub fn run(args: I) -> error::Result<()> where } } +fn build_spec(matches: &clap::ArgMatches) -> error::Result<()> { + let spec = load_spec(&matches)?; + info!("Building chain spec"); + let json = spec.to_json(matches.is_present("raw"))?; + print!("{}", json); + Ok(()) +} + +fn export_blocks(matches: &clap::ArgMatches) -> error::Result<()> { + let base_path = base_path(matches); + let spec = load_spec(&matches)?; + let mut config = service::Configuration::default_with_spec(spec); + config.database_path = db_path(&base_path).to_string_lossy().into(); + info!("DB path: {}", config.database_path); + let client = service::new_client(config)?; + let (exit_send, exit) = std::sync::mpsc::channel(); + ctrlc::CtrlC::set_handler(move || { + exit_send.clone().send(()).expect("Error sending exit notification"); + }); + info!("Exporting blocks"); + let mut block: u32 = match matches.value_of("from") { + Some(v) => v.parse().map_err(|_| "Invalid --from argument")?, + None => 1, + }; + + let last = match matches.value_of("to") { + Some(v) => v.parse().map_err(|_| "Invalid --to argument")?, + None => client.info()?.chain.best_number as u32, + }; + + if last < block { + return Err("Invalid block range specified".into()); + } + + let json = matches.is_present("json"); + + let mut file: Box = match matches.value_of("OUTPUT") { + Some(filename) => Box::new(File::open(filename)?), + None => Box::new(stdout()), + }; + + if !json { + file.write(&(last - block + 1).encode())?; + } + + loop { + if exit.try_recv().is_ok() { + break; + } + match client.block(&BlockId::number(block as u64))? { + Some(block) => { + if json { + serde_json::to_writer(&mut *file, &block).map_err(|e| format!("Eror writing JSON: {}", e))?; + } else { + file.write(&block.encode())?; + } + }, + None => break, + } + if block % 10000 == 0 { + info!("#{}", block); + } + if block == last { + break; + } + block += 1; + } + Ok(()) +} + +fn import_blocks(matches: &clap::ArgMatches) -> error::Result<()> { + let spec = load_spec(&matches)?; + let base_path = base_path(matches); + let mut config = service::Configuration::default_with_spec(spec); + config.database_path = db_path(&base_path).to_string_lossy().into(); + let client = service::new_client(config)?; + let (exit_send, exit) = std::sync::mpsc::channel(); + ctrlc::CtrlC::set_handler(move || { + exit_send.clone().send(()).expect("Error sending exit notification"); + }); + + let mut file: Box = match matches.value_of("INPUT") { + Some(filename) => Box::new(File::open(filename)?), + None => Box::new(stdin()), + }; + + info!("Importing blocks"); + let count: u32 = Slicable::decode(&mut file).ok_or("Error reading file")?; + let mut block = 0; + for _ in 0 .. count { + if exit.try_recv().is_ok() { + break; + } + match SignedBlock::decode(&mut file) { + Some(block) => { + let header = client.check_justification(block.block.header, block.justification.into())?; + client.import_block(BlockOrigin::File, header, Some(block.block.extrinsics))?; + }, + None => { + warn!("Error reading block data."); + break; + } + } + block += 1; + if block % 10000 == 0 { + info!("#{}", block); + } + } + info!("Imported {} blocks. Best: #{}", block, client.info()?.chain.best_number); + + Ok(()) +} + fn run_until_exit(mut core: reactor::Core, service: service::Service, matches: &clap::ArgMatches, sys_conf: SystemConfiguration) -> error::Result<()> where C: service::Components, diff --git a/polkadot/service/src/lib.rs b/polkadot/service/src/lib.rs index 8e2485c2fed23..7e627129177b4 100644 --- a/polkadot/service/src/lib.rs +++ b/polkadot/service/src/lib.rs @@ -99,6 +99,25 @@ pub fn new_full(config: Configuration) -> Result Result::Backend, + ::Executor, + Block>>, + error::Error> +{ + let db_settings = client_db::DatabaseSettings { + cache_size: None, + path: config.database_path.into(), + pruning: config.pruning, + }; + let executor = polkadot_executor::Executor::new(); + let is_validator = (config.roles & Role::VALIDATOR) == Role::VALIDATOR; + let components = components::FullComponents { is_validator }; + let (client, _) = components.build_client(db_settings, executor, &config.chain_spec)?; + Ok(client) +} + impl Service where Components: components::Components, diff --git a/substrate/client/src/client.rs b/substrate/client/src/client.rs index 2d8b18f93228c..e63933d8975a6 100644 --- a/substrate/client/src/client.rs +++ b/substrate/client/src/client.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use futures::sync::mpsc; use parking_lot::{Mutex, RwLock}; use primitives::AuthorityId; -use runtime_primitives::{bft::Justification, generic::BlockId}; +use runtime_primitives::{bft::Justification, generic::{BlockId, SignedBlock, Block as RuntimeBlock}}; use runtime_primitives::traits::{Block as BlockT, Header as HeaderT, Zero, One}; use runtime_primitives::BuildStorage; use primitives::storage::{StorageKey, StorageData}; @@ -416,6 +416,15 @@ impl Client where self.backend.blockchain().justification(*id) } + /// Get full block by id. + pub fn block(&self, id: &BlockId) -> error::Result>> { + Ok(match (self.header(id)?, self.body(id)?, self.justification(id)?) { + (Some(header), Some(extrinsics), Some(justification)) => + Some(SignedBlock { block: RuntimeBlock { header, extrinsics }, justification }), + _ => None, + }) + } + /// Get best block header. pub fn best_block_header(&self) -> error::Result<::Header> { let info = self.backend.blockchain().info().map_err(|e| error::Error::from_blockchain(Box::new(e)))?; diff --git a/substrate/client/src/lib.rs b/substrate/client/src/lib.rs index 4bf79e16d51ce..187a43636b496 100644 --- a/substrate/client/src/lib.rs +++ b/substrate/client/src/lib.rs @@ -53,7 +53,7 @@ pub use client::{ new_in_mem, BlockStatus, BlockOrigin, BlockchainEventStream, BlockchainEvents, Client, ClientInfo, ChainHead, - ImportResult, + ImportResult, JustifiedHeader, }; pub use blockchain::Info as ChainInfo; pub use call_executor::{CallResult, CallExecutor, LocalCallExecutor}; diff --git a/substrate/codec/src/slicable.rs b/substrate/codec/src/slicable.rs index c6363d35025e6..da402c8375727 100644 --- a/substrate/codec/src/slicable.rs +++ b/substrate/codec/src/slicable.rs @@ -38,6 +38,7 @@ pub trait Input { } } +#[cfg(not(feature = "std"))] impl<'a> Input for &'a [u8] { fn read(&mut self, into: &mut [u8]) -> usize { let len = ::core::cmp::min(into.len(), self.len()); @@ -47,6 +48,13 @@ impl<'a> Input for &'a [u8] { } } +#[cfg(feature = "std")] +impl Input for R { + fn read(&mut self, into: &mut [u8]) -> usize { + (self as &mut ::std::io::Read).read(into).unwrap_or(0) + } +} + /// Trait that allows zero-copy read/write of value-references to/from slices in LE format. pub trait Slicable: Sized { /// Attempt to deserialise the value from input. diff --git a/substrate/primitives/src/authority_id.rs b/substrate/primitives/src/authority_id.rs index 7ec5fa06f4ddd..665a55873669a 100644 --- a/substrate/primitives/src/authority_id.rs +++ b/substrate/primitives/src/authority_id.rs @@ -25,8 +25,8 @@ use H256; pub struct AuthorityId(pub [u8; 32]); impl AuthorityId { - /// Create an id from byte slice. - pub fn from_slice(data: &[u8]) -> Self { + /// Create an id from a 32-byte slice. Panics with other lengths. + fn from_slice(data: &[u8]) -> Self { let mut r = [0u8; 32]; r.copy_from_slice(data); AuthorityId(r) diff --git a/substrate/runtime/primitives/src/generic.rs b/substrate/runtime/primitives/src/generic.rs index 982bf52b54572..6f8a46841c748 100644 --- a/substrate/runtime/primitives/src/generic.rs +++ b/substrate/runtime/primitives/src/generic.rs @@ -28,6 +28,7 @@ use runtime_support::AuxDispatchable; use traits::{self, Member, SimpleArithmetic, SimpleBitOps, MaybeDisplay, Block as BlockT, Header as HeaderT, Hashing as HashingT}; use rstd::ops; +use bft::Justification; /// Definition of something that the external world might want to say. #[derive(PartialEq, Eq, Clone)] @@ -476,6 +477,34 @@ where } } +/// Abstraction over a substrate block and justification. +#[derive(PartialEq, Eq, Clone)] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "std", serde(deny_unknown_fields))] +pub struct SignedBlock { + /// Full block. + pub block: Block, + /// Block header justification. + pub justification: Justification, +} + +impl Slicable for SignedBlock { + fn decode(input: &mut I) -> Option { + Some(SignedBlock { + block: Slicable::decode(input)?, + justification: Slicable::decode(input)?, + }) + } + + fn encode(&self) -> Vec { + let mut v: Vec = Vec::new(); + v.extend(self.block.encode()); + v.extend(self.justification.encode()); + v + } +} + #[cfg(test)] mod tests { use codec::Slicable;