From 83d14978b3d2017d962dd39e0d35a0deb22bc8ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Mon, 13 Aug 2018 12:52:39 +0200 Subject: [PATCH 1/4] Expose extrinsics in pool. --- Cargo.lock | 1 + polkadot/transaction-pool/src/lib.rs | 15 ++++++++++++++- substrate/extrinsic-pool/src/api.rs | 9 +++++++++ substrate/extrinsic-pool/src/pool.rs | 17 +++++++++++++++++ substrate/rpc-servers/Cargo.toml | 1 + substrate/rpc-servers/src/lib.rs | 8 +++++--- substrate/rpc/src/author/mod.rs | 18 +++++++++++++----- substrate/rpc/src/author/tests.rs | 5 +++++ substrate/service/src/lib.rs | 3 ++- 9 files changed, 67 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d376a322e086f..bf884cb2f3f04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2930,6 +2930,7 @@ dependencies = [ "jsonrpc-pubsub 8.0.1 (git+https://github.com/paritytech/jsonrpc.git)", "jsonrpc-ws-server 8.0.0 (git+https://github.com/paritytech/jsonrpc.git)", "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-rpc 0.1.0", "substrate-runtime-primitives 0.1.0", ] diff --git a/polkadot/transaction-pool/src/lib.rs b/polkadot/transaction-pool/src/lib.rs index 447d97d004595..8354757efac8d 100644 --- a/polkadot/transaction-pool/src/lib.rs +++ b/polkadot/transaction-pool/src/lib.rs @@ -38,7 +38,7 @@ mod error; use std::{ cmp::Ordering, - collections::HashMap, + collections::{BTreeMap, HashMap}, ops::Deref, sync::Arc, }; @@ -54,6 +54,7 @@ use extrinsic_pool::{ use polkadot_api::PolkadotApi; use primitives::{AccountId, BlockId, Hash, Index, UncheckedExtrinsic as FutureProofUncheckedExtrinsic}; use runtime::{Address, UncheckedExtrinsic}; +use substrate_primitives::Bytes; use substrate_runtime_primitives::traits::{Bounded, Checkable, Hash as HashT, BlakeTwo256}; pub use extrinsic_pool::txpool::{Options, Status, LightStatus, VerifiedTransaction as VerifiedTransactionOps}; @@ -406,6 +407,7 @@ impl ExtrinsicPool for Transact A: PolkadotApi, { type Error = Error; + type InPool = BTreeMap>; fn submit(&self, block: BlockId, xts: Vec) -> Result> { xts.into_iter() @@ -437,6 +439,17 @@ impl ExtrinsicPool for Transact fn import_notification_stream(&self) -> EventStream { self.inner.import_notification_stream() } + + fn all(&self) -> Self::InPool { + self.inner.all(|it| it.fold(Default::default(), |mut map: Self::InPool, tx| { + // Map with `null` key is not serializable, so we fallback to default accountId. + map.entry(tx.sender().unwrap_or_default()) + .or_insert_with(Vec::new) + // use bytes type to make it serialize nicer. + .push(Bytes(tx.primitive_extrinsic())); + map + })) + } } #[cfg(test)] diff --git a/substrate/extrinsic-pool/src/api.rs b/substrate/extrinsic-pool/src/api.rs index 0d8bcfbf223d0..148ab49e4f4d9 100644 --- a/substrate/extrinsic-pool/src/api.rs +++ b/substrate/extrinsic-pool/src/api.rs @@ -16,6 +16,9 @@ //! External API for extrinsic pool. +use std::fmt::Debug; + +use serde::{Serialize, de::DeserializeOwned}; use txpool; use futures::sync::mpsc; @@ -43,6 +46,9 @@ pub trait ExtrinsicPool: Send + Sync + 'static { /// Error type type Error: Error; + /// Pooled extrinsics + type InPool: Debug + Serialize + DeserializeOwned + Send + Sync + 'static; + /// Submit a collection of extrinsics to the pool. fn submit(&self, block: BlockId, xt: Vec) -> Result, Self::Error>; @@ -54,4 +60,7 @@ pub trait ExtrinsicPool: Send + Sync + 'static { /// Return an event stream of transactions imported to the pool. fn import_notification_stream(&self) -> EventStream; + + /// Return all extrinsics in the pool aggregated by the sender. + fn all(&self) -> Self::InPool; } diff --git a/substrate/extrinsic-pool/src/pool.rs b/substrate/extrinsic-pool/src/pool.rs index 488834398e6bb..d5eb175ea68f6 100644 --- a/substrate/extrinsic-pool/src/pool.rs +++ b/substrate/extrinsic-pool/src/pool.rs @@ -149,4 +149,21 @@ impl Pool where { f(self.pool.read().pending(ready)) } + + /// Retrieve all transactions in the pool. The transactions might be unordered. + pub fn all(&self, f: F) -> T where + // F: FnOnce(txpool::UnorderedIterator) -> T, + F: FnOnce(txpool::PendingIterator>) -> T, + { + // f(self.pool.read().unordered_pending(AlwaysReady)) + f(self.pool.read().pending(AlwaysReady)) + } +} + +/// A Readiness implementation that returns `Ready` for all transactions. +pub struct AlwaysReady; +impl txpool::Ready for AlwaysReady { + fn is_ready(&mut self, _tx: &VEx) -> txpool::Readiness { + txpool::Readiness::Ready + } } diff --git a/substrate/rpc-servers/Cargo.toml b/substrate/rpc-servers/Cargo.toml index dec067b409fb4..6b7038d44a628 100644 --- a/substrate/rpc-servers/Cargo.toml +++ b/substrate/rpc-servers/Cargo.toml @@ -9,5 +9,6 @@ jsonrpc-http-server = { git = "https://github.com/paritytech/jsonrpc.git" } jsonrpc-pubsub = { git = "https://github.com/paritytech/jsonrpc.git" } jsonrpc-ws-server = { git = "https://github.com/paritytech/jsonrpc.git" } log = "0.3" +serde = "1.0" substrate-rpc = { path = "../rpc", version = "0.1" } substrate-runtime-primitives = { path = "../runtime/primitives" } diff --git a/substrate/rpc-servers/src/lib.rs b/substrate/rpc-servers/src/lib.rs index 11fa9aa9b60af..7344a0453c186 100644 --- a/substrate/rpc-servers/src/lib.rs +++ b/substrate/rpc-servers/src/lib.rs @@ -24,6 +24,7 @@ extern crate jsonrpc_core as rpc; extern crate jsonrpc_http_server as http; extern crate jsonrpc_pubsub as pubsub; extern crate jsonrpc_ws_server as ws; +extern crate serde; extern crate substrate_runtime_primitives; #[macro_use] @@ -38,16 +39,17 @@ pub type HttpServer = http::Server; pub type WsServer = ws::Server; /// Construct rpc `IoHandler` -pub fn rpc_handler( +pub fn rpc_handler( state: S, chain: C, author: A, system: Y, ) -> RpcHandler where - Block: 'static, + Block: BlockT + 'static, + PendingExtrinsics: serde::Serialize + serde::de::DeserializeOwned + Send + Sync + 'static, S: apis::state::StateApi, C: apis::chain::ChainApi, - A: apis::author::AuthorApi, + A: apis::author::AuthorApi, Y: apis::system::SystemApi, { let mut io = pubsub::PubSubHandler::default(); diff --git a/substrate/rpc/src/author/mod.rs b/substrate/rpc/src/author/mod.rs index 9077abdc3ffd8..cc613dbc988be 100644 --- a/substrate/rpc/src/author/mod.rs +++ b/substrate/rpc/src/author/mod.rs @@ -41,7 +41,7 @@ use self::error::Result; build_rpc_trait! { /// Substrate authoring RPC API - pub trait AuthorApi { + pub trait AuthorApi { type Metadata; /// Submit extrinsic for inclusion in block. @@ -51,6 +51,10 @@ build_rpc_trait! { #[rpc(name = "author_submitExtrinsic")] fn submit_extrinsic(&self, Bytes) -> Result; + /// A list of pending extrinsics. + #[rpc(name = "author_pending")] + fn pending_extrinsics(&self) -> Result; + #[pubsub(name = "author_extrinsicUpdate")] { /// Submit an extrinsic to watch. #[rpc(name = "author_submitAndWatchExtrinsic")] @@ -60,7 +64,6 @@ build_rpc_trait! { #[rpc(name = "author_unwatchExtrinsic")] fn unwatch_extrinsic(&self, SubscriptionId) -> Result; } - } } @@ -85,12 +88,13 @@ impl Author { } } -impl AuthorApi for Author where +impl AuthorApi for Author where B: client::backend::Backend + Send + Sync + 'static, E: client::CallExecutor + Send + Sync + 'static, Block: traits::Block + 'static, - Hash: traits::MaybeSerializeDebug + Sync + Send + 'static, - P: ExtrinsicPool, Hash>, + Hash: traits::MaybeSerializeDebug + Send + Sync + 'static, + InPool: traits::MaybeSerializeDebug + Send + Sync + 'static, + P: ExtrinsicPool, Hash, InPool=InPool>, P::Error: 'static, Ex: Codec, { @@ -145,4 +149,8 @@ impl AuthorApi for Author wh fn unwatch_extrinsic(&self, id: SubscriptionId) -> Result { Ok(self.subscriptions.cancel(id)) } + + fn pending_extrinsics(&self) -> Result { + Ok(self.pool.all()) + } } diff --git a/substrate/rpc/src/author/tests.rs b/substrate/rpc/src/author/tests.rs index 116ca7ea2c7bc..c0b54e929e716 100644 --- a/substrate/rpc/src/author/tests.rs +++ b/substrate/rpc/src/author/tests.rs @@ -46,6 +46,7 @@ impl fmt::Display for Error { impl api::ExtrinsicPool for DummyTxPool { type Error = Error; + type InPool = (); /// Submit extrinsic for inclusion in block. fn submit(&self, _block: BlockHash, xt: Vec) -> Result, Self::Error> { @@ -79,6 +80,10 @@ impl api::ExtrinsicPool for DummyTxPool { fn import_notification_stream(&self) -> api::EventStream { unreachable!() } + + fn all(&self) -> Vec { + vec![] + } } #[test] diff --git a/substrate/service/src/lib.rs b/substrate/service/src/lib.rs index 9fcd157f733c8..a265ec41e16a3 100644 --- a/substrate/service/src/lib.rs +++ b/substrate/service/src/lib.rs @@ -210,7 +210,8 @@ impl Service let chain = rpc::apis::chain::Chain::new(client.clone(), task_executor.clone()); let state = rpc::apis::state::State::new(client.clone(), task_executor.clone()); let author = rpc::apis::author::Author::new(client.clone(), extrinsic_pool.api(), task_executor.clone()); - rpc::rpc_handler::, _, _, _, _>( + + rpc::rpc_handler::, _, _, _, _, _>( state, chain, author, From d2b8fafd48392927602666dc55760c76bf1c7c3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 14 Aug 2018 09:56:44 +0200 Subject: [PATCH 2/4] Add test. --- demo/cli/src/lib.rs | 7 ++++++- substrate/rpc/src/author/mod.rs | 13 +++++++------ substrate/rpc/src/author/tests.rs | 21 ++++++++++++++++++--- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/demo/cli/src/lib.rs b/demo/cli/src/lib.rs index 7a8869db8410a..27a317e45fb8e 100644 --- a/demo/cli/src/lib.rs +++ b/demo/cli/src/lib.rs @@ -59,6 +59,7 @@ use demo_executor::NativeExecutor; struct DummyPool; impl extrinsic_pool::api::ExtrinsicPool for DummyPool { type Error = extrinsic_pool::txpool::Error; + type InPool = (); fn submit(&self, _block: BlockId, _: Vec) -> Result, Self::Error> @@ -79,6 +80,10 @@ impl extrinsic_pool::api::ExtrinsicPool for D fn import_notification_stream(&self) -> extrinsic_pool::api::EventStream { unreachable!() } + + fn all(&self) -> Self::InPool { + unreachable!() + } } struct DummySystem; @@ -176,7 +181,7 @@ pub fn run(args: I) -> error::Result<()> where let state = rpc::apis::state::State::new(client.clone(), runtime.executor()); let chain = rpc::apis::chain::Chain::new(client.clone(), runtime.executor()); let author = rpc::apis::author::Author::new(client.clone(), Arc::new(DummyPool), runtime.executor()); - rpc::rpc_handler::(state, chain, author, DummySystem) + rpc::rpc_handler::(state, chain, author, DummySystem) }; let http_address = "127.0.0.1:9933".parse().unwrap(); let ws_address = "127.0.0.1:9944".parse().unwrap(); diff --git a/substrate/rpc/src/author/mod.rs b/substrate/rpc/src/author/mod.rs index cc613dbc988be..1d8385c0e9e28 100644 --- a/substrate/rpc/src/author/mod.rs +++ b/substrate/rpc/src/author/mod.rs @@ -51,8 +51,8 @@ build_rpc_trait! { #[rpc(name = "author_submitExtrinsic")] fn submit_extrinsic(&self, Bytes) -> Result; - /// A list of pending extrinsics. - #[rpc(name = "author_pending")] + /// Returns all pending extrinsics, potentially grouped by sender. + #[rpc(name = "author_pendingExtrinsics")] fn pending_extrinsics(&self) -> Result; #[pubsub(name = "author_extrinsicUpdate")] { @@ -116,6 +116,10 @@ impl AuthorApi for Author Result { + Ok(self.pool.all()) + } + fn watch_extrinsic(&self, _metadata: Self::Metadata, subscriber: pubsub::Subscriber>, xt: Bytes) { let submit = || -> Result<_> { @@ -149,8 +153,5 @@ impl AuthorApi for Author Result { Ok(self.subscriptions.cancel(id)) } - - fn pending_extrinsics(&self) -> Result { - Ok(self.pool.all()) - } } + diff --git a/substrate/rpc/src/author/tests.rs b/substrate/rpc/src/author/tests.rs index c0b54e929e716..ebca4596adb7c 100644 --- a/substrate/rpc/src/author/tests.rs +++ b/substrate/rpc/src/author/tests.rs @@ -46,7 +46,7 @@ impl fmt::Display for Error { impl api::ExtrinsicPool for DummyTxPool { type Error = Error; - type InPool = (); + type InPool = Vec; /// Submit extrinsic for inclusion in block. fn submit(&self, _block: BlockHash, xt: Vec) -> Result, Self::Error> { @@ -81,8 +81,8 @@ impl api::ExtrinsicPool for DummyTxPool { unreachable!() } - fn all(&self) -> Vec { - vec![] + fn all(&self) -> Self::InPool { + vec![1, 2, 3, 4, 5] } } @@ -148,3 +148,18 @@ fn should_watch_extrinsic() { Some(r#"{"jsonrpc":"2.0","method":"test","params":{"result":{"usurped":5},"subscription":0}}"#.into()) ); } + +#[test] +fn should_return_pending_extrinsics() { + let runtime = runtime::Runtime::new().unwrap(); + let p = Author { + client: Arc::new(test_client::new()), + pool: Arc::new(DummyTxPool::default()), + subscriptions: Subscriptions::new(runtime.executor()), + }; + + assert_matches!( + p.pending_extrinsics(), + Ok(ref expected) if expected == &[1u8, 2, 3, 4, 5] + ); +} From 7dbad411742f1c362f35b4cc7b7e632c53572b6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 14 Aug 2018 10:13:03 +0200 Subject: [PATCH 3/4] Use latest transaction pool. --- Cargo.lock | 10 +++++----- substrate/extrinsic-pool/src/pool.rs | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf884cb2f3f04..667ec9121ba41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2216,7 +2216,7 @@ dependencies = [ "lazy_static 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.41 (registry+https://github.com/rust-lang/crates.io-index)", "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2787,7 +2787,7 @@ dependencies = [ "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.70 (registry+https://github.com/rust-lang/crates.io-index)", - "transaction-pool 1.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "transaction-pool 1.12.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3633,11 +3633,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "transaction-pool" -version = "1.12.1" +version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "error-chain 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "trace-time 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -4242,7 +4242,7 @@ dependencies = [ "checksum tokio-udp 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "137bda266504893ac4774e0ec4c2108f7ccdbcb7ac8dced6305fe9e4e0b5041a" "checksum trace-time 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5aea07da6582e957c6e737eeb63a5af79e648eeeaaaba8fd9a417f1124bafa41" "checksum traitobject 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "efd1f82c56340fdf16f2a953d7bda4f8fdffba13d93b00844c25572110b26079" -"checksum transaction-pool 1.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be1efb673ddf49ab4a99893eb3af02f6563636033fb832c2b7f937641ad62b17" +"checksum transaction-pool 1.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0f1fc049d9f039d3e9264f97f978309b94b5a5c56a5c18d28f91f469cef2b367" "checksum triehash 0.1.0 (git+https://github.com/paritytech/parity.git)" = "" "checksum triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "2033893a813c70e7d8a739ca6c36dc0a7a2c913ec718d7cbf84a3837bbe3c7ce" "checksum try-lock 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee2aa4715743892880f70885373966c83d73ef1b0838a664ef0c76fffd35e7c2" diff --git a/substrate/extrinsic-pool/src/pool.rs b/substrate/extrinsic-pool/src/pool.rs index d5eb175ea68f6..3ac2069f01d2f 100644 --- a/substrate/extrinsic-pool/src/pool.rs +++ b/substrate/extrinsic-pool/src/pool.rs @@ -152,11 +152,9 @@ impl Pool where /// Retrieve all transactions in the pool. The transactions might be unordered. pub fn all(&self, f: F) -> T where - // F: FnOnce(txpool::UnorderedIterator) -> T, - F: FnOnce(txpool::PendingIterator>) -> T, + F: FnOnce(txpool::UnorderedIterator) -> T, { - // f(self.pool.read().unordered_pending(AlwaysReady)) - f(self.pool.read().pending(AlwaysReady)) + f(self.pool.read().unordered_pending(AlwaysReady)) } } From 85a022db97baa77dc1b90466bb4280de3ade4b83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Drwi=C4=99ga?= Date: Tue, 14 Aug 2018 14:58:33 +0200 Subject: [PATCH 4/4] Fix compilation. --- polkadot/transaction-pool/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/polkadot/transaction-pool/src/lib.rs b/polkadot/transaction-pool/src/lib.rs index eb231cecdc3d2..f589ecfad00c6 100644 --- a/polkadot/transaction-pool/src/lib.rs +++ b/polkadot/transaction-pool/src/lib.rs @@ -178,9 +178,14 @@ impl txpool::Scoring for Scoring { } } - fn should_replace(&self, old: &VerifiedTransaction, _new: &VerifiedTransaction) -> bool { - // Always replace not fully verified transactions. - !old.is_fully_verified() + fn should_replace(&self, old: &VerifiedTransaction, _new: &VerifiedTransaction) -> Choice { + if old.is_fully_verified() { + // Don't allow new transactions if we are reaching the limit. + Choice::RejectNew + } else { + // Always replace not fully verified transactions. + Choice::ReplaceOld + } } }