diff --git a/Cargo.lock b/Cargo.lock index b0797c9dca..4b51cd8e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -431,11 +431,13 @@ dependencies = [ name = "browser-utils" version = "0.8.0" dependencies = [ + "chrono", "clear_on_drop", "console_error_panic_hook", "console_log", "futures 0.1.29", "futures 0.3.4", + "futures-timer 3.0.1", "js-sys", "kvdb-web", "libp2p", @@ -443,6 +445,7 @@ dependencies = [ "rand 0.6.5", "rand 0.7.3", "sc-chain-spec", + "sc-informant", "sc-network", "sc-service", "wasm-bindgen", @@ -478,9 +481,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.1.2" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb8038c1ddc0a5f73787b130f4cc75151e96ed33e417fde765eb5a81e3532f4" +checksum = "1f359dc14ff8911330a51ef78022d376f25ed00248912803b58f00cb1c27f742" [[package]] name = "byte-slice-cast" @@ -601,10 +604,12 @@ version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31850b4a4d6bae316f7a09e691c944c28299298837edc0a03f755618c23cbc01" dependencies = [ + "js-sys", "num-integer", "num-traits", "serde", "time", + "wasm-bindgen", ] [[package]] @@ -724,24 +729,25 @@ checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" [[package]] name = "cranelift-bforest" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd05aac8cefcde54ce26178df8f36cb1f518ac691db650e7d2440c2b6b41c4dc" +checksum = "fd0f53d59dc9ab1c8ab68c991d8406b52b7a0aab0b15b05a3a6895579c4e5dd9" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c63d9b6ff8a94f98deabab21880d7fd54996e0e16be687b6f80a3b6bdd9c188d" +checksum = "0381a794836fb994c47006465d46d46be072483b667f36013d993b9895117fee" dependencies = [ "byteorder 1.3.4", "cranelift-bforest", "cranelift-codegen-meta", "cranelift-codegen-shared", "cranelift-entity", + "gimli 0.20.0", "log 0.4.8", "serde", "smallvec 1.2.0", @@ -751,9 +757,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cb3df51c2c07d719d02869bfac6cabd8d82ee308d5b29ca62e6528723cc33a4" +checksum = "208c3c8d82bfef32a534c5020c6cfc3bc92f41388f1246b7bb98cf543331abaa" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -761,24 +767,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758f9426b2e22bf83fc1a6b231a9d53cd4830751883c7f0e196ebb3c210467b3" +checksum = "ea048c456a517e56fd6df8f0e3947922897e6e6f61fbc5eb557a36c7b8ff6394" [[package]] name = "cranelift-entity" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff064733df8b98f453060264a8790393d1e807aca6942706b42f79a4f7aae9ed" +checksum = "0c8c7ed50812194c9e9de1fa39c77b39fc9ab48173d5e7ee88b25b6a8953e9b8" dependencies = [ "serde", ] [[package]] name = "cranelift-frontend" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1eaafb5fa623dcbe19a28084a8226d7a1b17184a949c1a1f29a46b479867998d" +checksum = "21ceb931d9f919731df1b1ecdc716b5c66384b413a7f95909d1f45441ab9bef5" dependencies = [ "cranelift-codegen", "log 0.4.8", @@ -788,9 +794,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90033dbd7293f6fad4cf9dcd769cd621d60df22b1c5a11799e86359b7447a51d" +checksum = "564ee82268bc25b914fcf331edfc2452f2d9ca34f976b187b4ca668beba250c8" dependencies = [ "cranelift-codegen", "raw-cpuid", @@ -799,9 +805,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.50.0" +version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cb82a1071f88822763a583ec1a8688ffe5e2cda02c111d4483dd4376ed14d8" +checksum = "de63e2271b374be5b07f359184e2126a08fb24d24a740cbc178b7e0107ddafa5" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -809,7 +815,7 @@ dependencies = [ "log 0.4.8", "serde", "thiserror", - "wasmparser", + "wasmparser 0.48.2", ] [[package]] @@ -1247,9 +1253,9 @@ dependencies = [ [[package]] name = "evm" -version = "0.14.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f887b371f9999682ccc5b1cb771e7d4408ae61e93fc0343ceaeb761fca42d1" +checksum = "272f65e18a2b6449b682bfcdf6c3ccc63db0b93898d89c0fb237548bbfc764a5" dependencies = [ "evm-core", "evm-gasometer", @@ -1262,18 +1268,18 @@ dependencies = [ [[package]] name = "evm-core" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bcde5af3d542874ddeb53de0919302d57586ea04b3f76f54d865f8a6cdc70ae" +checksum = "66534d42e13d50f9101bed87cb568fd5aa929c600c3c13f8dadbbf39f5635945" dependencies = [ "primitive-types", ] [[package]] name = "evm-gasometer" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82bc9f275cb59d2bcc05d85c98736ddfaba003a7ef7b73893fa7c1c1fab29dc" +checksum = "39bc5b592803ca644781fe2290b7305ea5182f7c9516d615ddfb2308c2cab639" dependencies = [ "evm-core", "evm-runtime", @@ -1282,9 +1288,9 @@ dependencies = [ [[package]] name = "evm-runtime" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dbbc89d29618c3722c17ba78ddf432d40ace8ee27e3f8b28b52a85921112e4b" +checksum = "389e4b447fb26971a9c76c8aa49c0ab435f8e46e8fc46e1bc4ebf01f3c2b428f" dependencies = [ "evm-core", "primitive-types", @@ -1302,9 +1308,9 @@ dependencies = [ [[package]] name = "faerie" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f902f2af041f6c7177a2a04f805687cdc71e69c7cbef059a2755d8923f4cd7a8" +checksum = "74b9ed6159e4a6212c61d9c6a86bee01876b192a64accecf58d5b5ae3b667b52" dependencies = [ "anyhow", "goblin", @@ -1593,6 +1599,12 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "fs_extra" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2a4a2034423744d2cc7ca2068453168dcdb82c438419e639a26bd87839c674" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -1737,20 +1749,19 @@ checksum = "7b0a34e53cf6cdcd0178aa573aed466b646eb3db769570841fda0c7ede375a27" [[package]] name = "futures-timer" -version = "0.4.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878f1d2fc31355fa02ed2372e741b0c17e58373341e6a122569b4623a14a7d33" -dependencies = [ - "futures-core-preview", - "futures-util-preview", - "pin-utils", -] +checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" [[package]] name = "futures-timer" -version = "2.0.2" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1de7508b218029b0f01662ed8f61b1c964b3ae99d6f25462d0f55a595109df6" +checksum = "3de1a2b2a2a33d9e60e17980b60ee061eeaae96a5abe9121db0fdb9af167a1c5" +dependencies = [ + "gloo-timers", + "send_wrapper 0.4.0", +] [[package]] name = "futures-util" @@ -1868,6 +1879,16 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "gimli" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dd6190aad0f05ddbbf3245c54ed14ca4aa6dd32f22312b70d8f168c3e3e633" +dependencies = [ + "byteorder 1.3.4", + "indexmap", +] + [[package]] name = "glob" version = "0.2.11" @@ -1893,6 +1914,19 @@ dependencies = [ "regex", ] +[[package]] +name = "gloo-timers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b2d17dbd803c2fc86cb1b613adf63192046a7176f383a8302594654752c4c4a" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "goblin" version = "0.1.3" @@ -1911,7 +1945,7 @@ dependencies = [ "async-std", "chrono", "derive_more", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "futures-util", "hyper 0.13.2", "lazy_static", @@ -1927,7 +1961,7 @@ name = "grafana-data-source-test" version = "2.0.0" dependencies = [ "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "grafana-data-source", "rand 0.7.3", ] @@ -2030,9 +2064,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" +checksum = "76cdda6bf525062a0c9e8f14ee2b37935c86b8efb6c8b69b3c83dfb518a914af" [[package]] name = "hex-literal" @@ -2429,28 +2463,28 @@ dependencies = [ [[package]] name = "jsonrpc-http-server" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d83d348120edee487c560b7cdd2565055d61cda053aa0d0ef0f8b6a18429048" +checksum = "816d63997ea45d3634608edbef83ddb35e661f7c0b27b5b72f237e321f0e9807" dependencies = [ "hyper 0.12.35", "jsonrpc-core", "jsonrpc-server-utils", "log 0.4.8", "net2", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "unicase 2.6.0", ] [[package]] name = "jsonrpc-pubsub" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3453625f0f0f5cd6d6776d389d73b7d70fcc98620b7cbb1cbbb1f6a36e95f39a" +checksum = "5b31c9b90731276fdd24d896f31bb10aecf2e5151733364ae81123186643d939" dependencies = [ "jsonrpc-core", "log 0.4.8", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "serde", ] @@ -2472,14 +2506,14 @@ dependencies = [ [[package]] name = "jsonrpc-ws-server" -version = "14.0.5" +version = "14.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34faa167c3ac9705aeecb986c0da6056529f348425dbe0441db60a2c4cc41d1" +checksum = "b94e5773b2ae66e0e02c80775ce6bbba6f15d5bb47c14ec36a36fcf94f8df851" dependencies = [ "jsonrpc-core", "jsonrpc-server-utils", "log 0.4.8", - "parking_lot 0.9.0", + "parking_lot 0.10.0", "slab", "ws", ] @@ -2596,6 +2630,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b294d6fa9ee409a054354afc4352b0b9ef7ca222c69b8812cbea9e7d2bf3783f" +[[package]] +name = "leb128" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3576a87f2ba00f6f106fdfcd16db1d698d648a26ad8e0573cad8537c3c362d2a" + [[package]] name = "libc" version = "0.2.66" @@ -3436,7 +3476,7 @@ dependencies = [ "sp-state-machine", "sp-trie", "substrate-test-client", - "trie-root 0.15.2", + "trie-root", "wabt", ] @@ -3607,8 +3647,12 @@ dependencies = [ name = "node-testing" version = "2.0.0" dependencies = [ + "criterion 0.3.1", "frame-support", "frame-system", + "fs_extra", + "hex-literal", + "log 0.4.8", "node-executor", "node-primitives", "node-runtime", @@ -3623,13 +3667,25 @@ dependencies = [ "pallet-transaction-payment", "pallet-treasury", "parity-scale-codec", + "sc-cli", "sc-client", + "sc-client-api", + "sc-client-db", "sc-executor", + "sc-service", + "sp-api", + "sp-block-builder", + "sp-blockchain", + "sp-consensus", "sp-core", + "sp-finality-tracker", + "sp-inherents", "sp-io", "sp-keyring", "sp-runtime", + "sp-timestamp", "substrate-test-client", + "tempdir", "wabt", ] @@ -5497,7 +5553,7 @@ dependencies = [ "derive_more", "env_logger 0.7.1", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -5601,6 +5657,7 @@ dependencies = [ "regex", "rpassword", "sc-client-api", + "sc-informant", "sc-network", "sc-service", "sc-telemetry", @@ -5701,6 +5758,7 @@ dependencies = [ "parity-util-mem", "parking_lot 0.10.0", "quickcheck", + "rand 0.7.3", "sc-client", "sc-client-api", "sc-executor", @@ -5724,7 +5782,7 @@ dependencies = [ "env_logger 0.7.1", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5764,7 +5822,7 @@ dependencies = [ "fork-tree", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "log 0.4.8", "merlin", "num-bigint", @@ -5870,7 +5928,7 @@ name = "sc-consensus-slots" version = "0.8.0" dependencies = [ "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -5967,11 +6025,6 @@ name = "sc-executor-wasmtime" version = "0.8.0" dependencies = [ "assert_matches", - "cranelift-codegen", - "cranelift-entity", - "cranelift-frontend", - "cranelift-native", - "cranelift-wasm", "log 0.4.8", "parity-scale-codec", "parity-wasm 0.41.0", @@ -5981,9 +6034,7 @@ dependencies = [ "sp-runtime-interface", "sp-wasm-interface", "wasmi", - "wasmtime-environ", - "wasmtime-jit", - "wasmtime-runtime", + "wasmtime", ] [[package]] @@ -5996,7 +6047,7 @@ dependencies = [ "fork-tree", "futures 0.1.29", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "log 0.4.8", "parity-scale-codec", "parking_lot 0.10.0", @@ -6027,6 +6078,22 @@ dependencies = [ "tokio 0.1.22", ] +[[package]] +name = "sc-informant" +version = "0.8.0" +dependencies = [ + "ansi_term 0.12.1", + "futures 0.3.4", + "log 0.4.8", + "parity-util-mem", + "sc-client-api", + "sc-network", + "sc-service", + "sp-blockchain", + "sp-runtime", + "wasm-timer", +] + [[package]] name = "sc-keystore" version = "2.0.0" @@ -6046,6 +6113,8 @@ dependencies = [ name = "sc-network" version = "0.8.0" dependencies = [ + "assert_matches", + "async-std", "bitflags", "bytes 0.5.4", "derive_more", @@ -6055,15 +6124,18 @@ dependencies = [ "fnv", "fork-tree", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "futures_codec", "libp2p", "linked-hash-map", "linked_hash_set", "log 0.4.8", "lru 0.4.3", + "nohash-hasher", "parity-scale-codec", "parking_lot 0.10.0", + "prost", + "prost-build", "quickcheck", "rand 0.7.3", "rustc-hex", @@ -6085,10 +6157,14 @@ dependencies = [ "sp-runtime", "sp-test-primitives", "substrate-test-client", + "substrate-test-runtime", "substrate-test-runtime-client", "tempfile", + "thiserror", "unsigned-varint", "void", + "wasm-timer", + "yamux", "zeroize 1.1.0", ] @@ -6098,13 +6174,14 @@ version = "0.8.0" dependencies = [ "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "lru 0.1.17", "parking_lot 0.10.0", "sc-network", "sp-runtime", + "wasm-timer", ] [[package]] @@ -6114,7 +6191,7 @@ dependencies = [ "env_logger 0.7.1", "futures 0.1.29", "futures 0.3.4", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6143,7 +6220,7 @@ dependencies = [ "fnv", "futures 0.1.29", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "hyper 0.12.35", "hyper-rustls", "log 0.4.8", @@ -6175,6 +6252,7 @@ dependencies = [ "log 0.4.8", "rand 0.7.3", "serde_json", + "wasm-timer", ] [[package]] @@ -6231,6 +6309,7 @@ dependencies = [ "serde_json", "sp-core", "sp-rpc", + "sp-runtime", "sp-transaction-pool", "sp-version", ] @@ -6271,7 +6350,7 @@ dependencies = [ "futures 0.1.29", "futures 0.3.4", "futures-diagnose", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "grafana-data-source", "lazy_static", "log 0.4.8", @@ -6313,6 +6392,7 @@ dependencies = [ "tokio 0.2.11", "tokio-executor 0.1.10", "tracing", + "wasm-timer", ] [[package]] @@ -6352,7 +6432,7 @@ version = "2.0.0" dependencies = [ "bytes 0.5.4", "futures 0.3.4", - "futures-timer 2.0.2", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parking_lot 0.10.0", @@ -6364,6 +6444,7 @@ dependencies = [ "slog-scope", "take_mut", "void", + "wasm-timer", ] [[package]] @@ -6390,15 +6471,18 @@ dependencies = [ "criterion 0.3.1", "derive_more", "futures 0.3.4", + "linked-hash-map", "log 0.4.8", "parity-scale-codec", "parity-util-mem", "parking_lot 0.10.0", "serde", + "sp-blockchain", "sp-core", "sp-runtime", "sp-transaction-pool", "substrate-test-runtime", + "wasm-timer", ] [[package]] @@ -6422,6 +6506,7 @@ dependencies = [ "sp-transaction-pool", "substrate-test-runtime-client", "substrate-test-runtime-transaction-pool", + "wasm-timer", ] [[package]] @@ -6569,6 +6654,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "686ef91cf020ad8d4aca9a7047641fd6add626b7b89e14546c2b6a76781cf822" +[[package]] +name = "send_wrapper" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f638d531eccd6e23b980caf34876660d38e265409d8e99b397ab71eb3612fad0" + [[package]] name = "serde" version = "1.0.104" @@ -6591,9 +6682,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b01d7f0288608a01dca632cf1df859df6fd6ffa885300fc275ce2ba6221953" +checksum = "15913895b61e0be854afd32fd4163fcd2a3df34142cf2cb961b310ce694cbf90" dependencies = [ "itoa", "ryu", @@ -6918,7 +7009,7 @@ dependencies = [ "derive_more", "futures 0.3.4", "futures-diagnose", - "futures-timer 0.4.0", + "futures-timer 3.0.1", "libp2p", "log 0.4.8", "parity-scale-codec", @@ -7260,8 +7351,8 @@ dependencies = [ "sp-externalities", "sp-panic-handler", "sp-trie", - "trie-db 0.19.2", - "trie-root 0.15.2", + "trie-db", + "trie-root", ] [[package]] @@ -7300,6 +7391,7 @@ dependencies = [ "sp-inherents", "sp-runtime", "sp-std", + "wasm-timer", ] [[package]] @@ -7327,8 +7419,8 @@ dependencies = [ "sp-core", "sp-std", "trie-bench", - "trie-db 0.20.0", - "trie-root 0.16.0", + "trie-db", + "trie-root", "trie-standardmap", ] @@ -7462,6 +7554,7 @@ dependencies = [ "hyper 0.12.35", "itertools", "jsonrpc-core-client", + "libp2p", "node-primitives", "node-runtime", "pallet-balances", @@ -7471,6 +7564,7 @@ dependencies = [ "rpassword", "rustc-hex", "sc-rpc", + "serde_json", "sp-core", "sp-runtime", "substrate-bip39", @@ -7589,7 +7683,7 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "substrate-wasm-builder-runner", - "trie-db 0.20.0", + "trie-db", ] [[package]] @@ -7618,6 +7712,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.10.0", "sc-transaction-graph", + "sp-blockchain", "sp-runtime", "sp-transaction-pool", "substrate-test-runtime-client", @@ -7712,9 +7807,9 @@ checksum = "f764005d11ee5f36500a149ace24e00e3da98b0158b3e2d53a7495660d3f4d60" [[package]] name = "target-lexicon" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4c118a7a38378f305a9e111fcb2f7f838c0be324bfb31a77ea04f7f6e684b4" +checksum = "ab0e7238dcc7b40a7be719a25365910f6807bd864f4cce6b2e6b873658e2b19d" [[package]] name = "target_info" @@ -7722,6 +7817,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c63f48baada5c52e65a29eef93ab4f8982681b67f9e8d29c7b05abcfec2b9ffe" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.1.0" @@ -8212,25 +8317,11 @@ dependencies = [ "keccak-hasher", "memory-db", "parity-scale-codec", - "trie-db 0.20.0", - "trie-root 0.16.0", + "trie-db", + "trie-root", "trie-standardmap", ] -[[package]] -name = "trie-db" -version = "0.19.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d747ae5b6f078df7e46477fcc7df66df9eb4f27a031cf4a7c890a8dd03d8e6" -dependencies = [ - "hash-db", - "hashbrown 0.6.3", - "log 0.4.8", - "rand 0.6.5", - "rustc-hex", - "smallvec 1.2.0", -] - [[package]] name = "trie-db" version = "0.20.0" @@ -8244,15 +8335,6 @@ dependencies = [ "smallvec 1.2.0", ] -[[package]] -name = "trie-root" -version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b779f7c1c8fe9276365d9d5be5c4b5adeacf545117bb3f64c974305789c5c0b" -dependencies = [ - "hash-db", -] - [[package]] name = "trie-root" version = "0.16.0" @@ -8672,35 +8754,61 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.39.3" +version = "0.48.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c702914acda5feeeffbc29e4d953e5b9ce79d8b98da4dbf18a77086e116c5470" +checksum = "073da89bf1c84db000dd68ce660c1b4a08e3a2d28fd1e3394ab9e7abdde4a0f8" + +[[package]] +name = "wasmparser" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e41b27a1677fe28c115de49efca55dabb14f7fece2c32947ffb9b1064fe5bd4" + +[[package]] +name = "wasmtime" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5614d964c3e7d07a13b59aca66103c52656bd80430f0d86dc7eeb3af4f03d4a2" +dependencies = [ + "anyhow", + "backtrace", + "cfg-if", + "lazy_static", + "libc", + "region", + "rustc-demangle", + "target-lexicon", + "wasmparser 0.51.1", + "wasmtime-environ", + "wasmtime-jit", + "wasmtime-runtime", + "wat", + "winapi 0.3.8", +] [[package]] name = "wasmtime-debug" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5008729ad53f75020f28fa0d682269335d6f0eac0b3ffafe31f185b2f33aca74" +checksum = "feb5900275b4ef0b621ce725b9d5660b12825d7f7d79b392b97baf089ffab8c0" dependencies = [ "anyhow", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", "faerie", - "gimli", + "gimli 0.19.0", "more-asserts", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.51.1", "wasmtime-environ", ] [[package]] name = "wasmtime-environ" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3947662a0b8e05b1418465e64f16de9114f9fec18cc3f56e0ed5aa7737b89d0" +checksum = "f04661851e133fb11691c4a0f92a705766b4bbf7afc06811f949e295cc8414fc" dependencies = [ + "anyhow", "base64 0.11.0", "bincode", "cranelift-codegen", @@ -8710,37 +8818,37 @@ dependencies = [ "errno", "file-per-thread-logger", "indexmap", - "lazy_static", "libc", "log 0.4.8", "more-asserts", "rayon", "serde", "sha2", - "spin", "thiserror", "toml", - "wasmparser", + "wasmparser 0.51.1", "winapi 0.3.8", "zstd", ] [[package]] name = "wasmtime-jit" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed7922689461a7b5bd0d9c7350cac526c8a520a23b3ffd7f5b446ac51dfc51f" +checksum = "d451353764ce55c9bb6a8b260063cfc209b7adadd277a9a872ab4563a69e357c" dependencies = [ "anyhow", + "cfg-if", "cranelift-codegen", "cranelift-entity", "cranelift-frontend", + "cranelift-native", "cranelift-wasm", "more-asserts", "region", "target-lexicon", "thiserror", - "wasmparser", + "wasmparser 0.51.1", "wasmtime-debug", "wasmtime-environ", "wasmtime-runtime", @@ -8749,16 +8857,14 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "0.8.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "781d6bb8b346efaa3dc39746386957cd79b8d841e8652ed9b02d77bcf64fb514" +checksum = "7dbd4fc114b828cae3e405fed413df4b3814d87a92ea029640cec9ba41f0c162" dependencies = [ + "backtrace", "cc", - "cranelift-codegen", - "cranelift-entity", - "cranelift-wasm", + "cfg-if", "indexmap", - "lazy_static", "libc", "memoffset", "more-asserts", @@ -8768,6 +8874,24 @@ dependencies = [ "winapi 0.3.8", ] +[[package]] +name = "wast" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12a729d076deb29c8509fa71f2d427729f9394f9496844ed8fcab152f35d163d" +dependencies = [ + "leb128", +] + +[[package]] +name = "wat" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5795e34a4b39893653dec97e644fac85c31398e0ce1abecc48967aac83d9e8ce" +dependencies = [ + "wast", +] + [[package]] name = "web-sys" version = "0.3.35" diff --git a/Cargo.toml b/Cargo.toml index 9eef2ec706..ebf85d7db1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ members = [ "client/executor/wasmtime", "client/executor/runtime-test", "client/finality-grandpa", + "client/informant", "client/tracing", "client/keystore", "client/network", diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index d113f90866..cf5cb361fc 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -159,7 +159,6 @@ pub fn new_full(config: Configuration) grandpa_link, service.network(), service.on_exit(), - service.spawn_task_handle(), )?); }, (true, false) => { @@ -172,7 +171,6 @@ pub fn new_full(config: Configuration) on_exit: service.on_exit(), telemetry_on_connect: Some(service.telemetry_on_connect_stream()), voting_rule: grandpa::VotingRulesBuilder::default().build(), - executor: service.spawn_task_handle(), }; // the GRANDPA voter task is considered infallible, i.e. diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 746a95bc46..506b69bf91 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -12,7 +12,7 @@ use sp_std::prelude::*; use sp_core::OpaqueMetadata; use sp_runtime::{ ApplyExtrinsicResult, transaction_validity::TransactionValidity, generic, create_runtime_str, - impl_opaque_keys, MultiSignature + impl_opaque_keys, MultiSignature, }; use sp_runtime::traits::{ BlakeTwo256, Block as BlockT, StaticLookup, Verify, ConvertInto, IdentifyAccount diff --git a/bin/node/cli/browser-demo/index.html b/bin/node/cli/browser-demo/index.html index 0b66b612f1..f40863c46e 100644 --- a/bin/node/cli/browser-demo/index.html +++ b/bin/node/cli/browser-demo/index.html @@ -15,11 +15,12 @@ async function start() { log('Loading WASM'); await init('./pkg/node_cli_bg.wasm'); - log('Successfully loaded WASM'); + log('Fetching chain spec'); + const chain_spec_response = await fetch("https://raw.githubusercontent.com/paritytech/substrate/master/bin/node/cli/res/flaming-fir.json"); + const chain_spec_text = await chain_spec_response.text(); // Build our client. - log('Starting client'); - let client = await start_client(ws()); + let client = await start_client(chain_spec_text, 'info', ws()); log('Client started'); client.rpcSubscribe('{"method":"chain_subscribeNewHead","params":[],"id":1,"jsonrpc":"2.0"}', diff --git a/bin/node/cli/src/browser.rs b/bin/node/cli/src/browser.rs index e05238b82d..80ca963445 100644 --- a/bin/node/cli/src/browser.rs +++ b/bin/node/cli/src/browser.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::ChainSpec; +use crate::chain_spec::ChainSpec; use log::info; use wasm_bindgen::prelude::*; use sc_service::Configuration; @@ -22,20 +22,20 @@ use browser_utils::{ Transport, Client, browser_configuration, set_console_error_panic_hook, init_console_log, }; +use std::str::FromStr; /// Starts the client. #[wasm_bindgen] -pub async fn start_client(wasm_ext: Transport) -> Result { - start_inner(wasm_ext) +pub async fn start_client(chain_spec: String, log_level: String, wasm_ext: Transport) -> Result { + start_inner(chain_spec, log_level, wasm_ext) .await .map_err(|err| JsValue::from_str(&err.to_string())) } -async fn start_inner(wasm_ext: Transport) -> Result> { +async fn start_inner(chain_spec: String, log_level: String, wasm_ext: Transport) -> Result> { set_console_error_panic_hook(); - init_console_log(log::Level::Info)?; - - let chain_spec = ChainSpec::FlamingFir.load() + init_console_log(log::Level::from_str(&log_level)?)?; + let chain_spec = ChainSpec::from_json_bytes(chain_spec.as_bytes().to_vec()) .map_err(|e| format!("{:?}", e))?; let config: Configuration<_, _> = browser_configuration(wasm_ext, chain_spec) diff --git a/bin/node/cli/src/chain_spec.rs b/bin/node/cli/src/chain_spec.rs index 4dedd21cd5..e108d86b98 100644 --- a/bin/node/cli/src/chain_spec.rs +++ b/bin/node/cli/src/chain_spec.rs @@ -257,13 +257,17 @@ pub fn testnet_genesis( }), pallet_democracy: Some(DemocracyConfig::default()), pallet_collective_Instance1: Some(CouncilConfig { - members: endowed_accounts.iter().cloned() - .collect::>()[..(num_endowed_accounts + 1) / 2].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), phantom: Default::default(), }), pallet_collective_Instance2: Some(TechnicalCommitteeConfig { - members: endowed_accounts.iter().cloned() - .collect::>()[..(num_endowed_accounts + 1) / 2].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), phantom: Default::default(), }), pallet_contracts: Some(ContractsConfig { @@ -291,7 +295,10 @@ pub fn testnet_genesis( pallet_membership_Instance1: Some(Default::default()), pallet_treasury: Some(Default::default()), pallet_society: Some(SocietyConfig { - members: endowed_accounts[0..3].to_vec(), + members: endowed_accounts.iter() + .take((num_endowed_accounts + 1) / 2) + .cloned() + .collect(), pot: 0, max_members: 999, }), @@ -355,6 +362,7 @@ pub(crate) mod tests { use super::*; use crate::service::{new_full, new_light}; use sc_service_test; + use sp_runtime::BuildStorage; fn local_testnet_genesis_instant_single() -> GenesisConfig { testnet_genesis( @@ -404,4 +412,19 @@ pub(crate) mod tests { |config| new_light(config), ); } + + #[test] + fn test_create_development_chain_spec() { + development_config().build_storage().unwrap(); + } + + #[test] + fn test_create_local_testnet_chain_spec() { + local_testnet_config().build_storage().unwrap(); + } + + #[test] + fn test_staging_test_net_chain_spec() { + staging_testnet_config().build_storage().unwrap(); + } } diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index dd3ee4a596..7844c2c0a5 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -52,34 +52,13 @@ pub enum Subcommand { /// Please note: this command currently only works on an empty database! #[derive(Debug, StructOpt, Clone)] pub struct FactoryCmd { - /// How often to repeat. This option only has an effect in mode `MasterToNToM`. - #[structopt(long="rounds", default_value = "1")] - pub rounds: u64, + /// Number of blocks to generate. + #[structopt(long="blocks", default_value = "1")] + pub blocks: u32, - /// MasterToN: Manufacture `num` transactions from the master account - /// to `num` randomly created accounts, one each. - /// - /// MasterTo1: Manufacture `num` transactions from the master account - /// to exactly one other randomly created account. - /// - /// MasterToNToM: Manufacture `num` transactions from the master account - /// to `num` randomly created accounts. - /// From each of these randomly created accounts manufacture - /// a transaction to another randomly created account. - /// Repeat this `rounds` times. If `rounds` = 1 the behavior - /// is the same as `MasterToN`.{n} - /// A -> B, A -> C, A -> D, ... x `num`{n} - /// B -> E, C -> F, D -> G, ...{n} - /// ... x `rounds` - /// - /// These three modes control manufacturing. - #[structopt(long="mode", default_value = "MasterToN")] - pub mode: node_transaction_factory::Mode, - - /// Number of transactions to generate. In mode `MasterNToNToM` this is - /// the number of transactions per round. - #[structopt(long="num", default_value = "8")] - pub num: u64, + /// Number of transactions to push per block. + #[structopt(long="transactions", default_value = "8")] + pub transactions: u32, #[allow(missing_docs)] #[structopt(flatten)] diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 3bfed148f5..61d1051796 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -67,9 +67,8 @@ where } let factory_state = FactoryState::new( - cli_args.mode.clone(), - cli_args.num, - cli_args.rounds, + cli_args.blocks, + cli_args.transactions, ); let service_builder = new_full_start!(config).0; diff --git a/bin/node/cli/src/factory_impl.rs b/bin/node/cli/src/factory_impl.rs index 05b2738887..2fd15c73d8 100644 --- a/bin/node/cli/src/factory_impl.rs +++ b/bin/node/cli/src/factory_impl.rs @@ -33,7 +33,6 @@ use sp_runtime::{ generic::Era, traits::{Block as BlockT, Header as HeaderT, SignedExtension, Verify, IdentifyAccount} }; use node_transaction_factory::RuntimeAdapter; -use node_transaction_factory::modes::Mode; use sp_inherents::InherentData; use sp_timestamp; use sp_finality_tracker; @@ -41,14 +40,10 @@ use sp_finality_tracker; type AccountPublic = ::Signer; pub struct FactoryState { - block_no: N, - - mode: Mode, - start_number: u32, - rounds: u32, - round: u32, - block_in_round: u32, - num: u32, + blocks: u32, + transactions: u32, + block_number: N, + index: u32, } type Number = <::Header as HeaderT>::Number; @@ -79,63 +74,35 @@ impl RuntimeAdapter for FactoryState { type Number = Number; fn new( - mode: Mode, - num: u64, - rounds: u64, + blocks: u32, + transactions: u32, ) -> FactoryState { FactoryState { - mode, - num: num as u32, - round: 0, - rounds: rounds as u32, - block_in_round: 0, - block_no: 0, - start_number: 0, + blocks, + transactions, + block_number: 0, + index: 0, } } - fn block_no(&self) -> Self::Number { - self.block_no - } - - fn block_in_round(&self) -> Self::Number { - self.block_in_round - } - - fn rounds(&self) -> Self::Number { - self.rounds - } - - fn num(&self) -> Self::Number { - self.num - } - - fn round(&self) -> Self::Number { - self.round - } - - fn start_number(&self) -> Self::Number { - self.start_number - } - - fn mode(&self) -> &Mode { - &self.mode + fn block_number(&self) -> u32 { + self.block_number } - fn set_block_no(&mut self, val: Self::Number) { - self.block_no = val; + fn blocks(&self) -> u32 { + self.blocks } - fn set_block_in_round(&mut self, val: Self::Number) { - self.block_in_round = val; + fn transactions(&self) -> u32 { + self.transactions } - fn set_round(&mut self, val: Self::Number) { - self.round = val; + fn set_block_number(&mut self, value: u32) { + self.block_number = value; } fn transfer_extrinsic( - &self, + &mut self, sender: &Self::AccountId, key: &Self::Secret, destination: &Self::AccountId, @@ -144,10 +111,12 @@ impl RuntimeAdapter for FactoryState { genesis_hash: &::Hash, prior_block_hash: &::Hash, ) -> ::Extrinsic { - let index = self.extract_index(&sender, prior_block_hash); - let phase = self.extract_phase(*prior_block_hash); + let phase = self.block_number() as Self::Phase; + let extra = Self::build_extra(self.index, phase); + self.index += 1; + sign::(CheckedExtrinsic { - signed: Some((sender.clone(), Self::build_extra(index, phase))), + signed: Some((sender.clone(), extra)), function: Call::Balances( BalancesCall::transfer( pallet_indices::address::Address::Id(destination.clone().into()), @@ -158,12 +127,12 @@ impl RuntimeAdapter for FactoryState { } fn inherent_extrinsics(&self) -> InherentData { - let timestamp = (self.block_no as u64 + 1) * MinimumPeriod::get(); + let timestamp = (self.block_number as u64 + 1) * MinimumPeriod::get(); let mut inherent = InherentData::new(); inherent.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) .expect("Failed putting timestamp inherent"); - inherent.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &self.block_no) + inherent.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &self.block_number) .expect("Failed putting finalized number inherent"); inherent } @@ -181,49 +150,16 @@ impl RuntimeAdapter for FactoryState { } /// Generates a random `AccountId` from `seed`. - fn gen_random_account_id(seed: &Self::Number) -> Self::AccountId { - let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(*seed)); + fn gen_random_account_id(seed: u32) -> Self::AccountId { + let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(seed)); AccountPublic::from(pair.public()).into_account() } /// Generates a random `Secret` from `seed`. - fn gen_random_account_secret(seed: &Self::Number) -> Self::Secret { - let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(*seed)); + fn gen_random_account_secret(seed: u32) -> Self::Secret { + let pair: sr25519::Pair = sr25519::Pair::from_seed(&gen_seed_bytes(seed)); pair } - - fn extract_index( - &self, - _account_id: &Self::AccountId, - _block_hash: &::Hash, - ) -> Self::Index { - // TODO get correct index for account via api. See #2587. - // This currently prevents the factory from being used - // without a preceding purge of the database. - if self.mode == Mode::MasterToN || self.mode == Mode::MasterTo1 { - self.block_no() as Self::Index - } else { - match self.round() { - 0 => - // if round is 0 all transactions will be done with master as a sender - self.block_no() as Self::Index, - _ => - // if round is e.g. 1 every sender account will be new and not yet have - // any transactions done - 0 - } - } - } - - fn extract_phase( - &self, - _block_hash: ::Hash - ) -> Self::Phase { - // TODO get correct phase via api. See #2587. - // This currently prevents the factory from being used - // without a preceding purge of the database. - self.block_no() as Self::Phase - } } fn gen_seed_bytes(seed: u32) -> [u8; 32] { diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index a90e558b2f..b3ec72de2b 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -216,7 +216,6 @@ macro_rules! new_full { grandpa_link, service.network(), service.on_exit(), - service.spawn_task_handle(), )?); }, (true, false) => { @@ -229,7 +228,6 @@ macro_rules! new_full { on_exit: service.on_exit(), telemetry_on_connect: Some(service.telemetry_on_connect_stream()), voting_rule: grandpa::VotingRulesBuilder::default().build(), - executor: service.spawn_task_handle(), }; // the GRANDPA voter task is considered infallible, i.e. // if it fails we take down the service with it. diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index 0d68b70522..1d894e39fa 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -15,7 +15,7 @@ sp-core = { version = "2.0.0", path = "../../../primitives/core" } sp-io = { version = "2.0.0", path = "../../../primitives/io" } sp-state-machine = { version = "0.8", path = "../../../primitives/state-machine" } sp-trie = { version = "2.0.0", path = "../../../primitives/trie" } -trie-root = "0.15.2" +trie-root = "0.16.0" [dev-dependencies] criterion = "0.3.0" diff --git a/bin/node/executor/src/lib.rs b/bin/node/executor/src/lib.rs index 812c018502..72f40b7c1f 100644 --- a/bin/node/executor/src/lib.rs +++ b/bin/node/executor/src/lib.rs @@ -25,5 +25,6 @@ use sc_executor::native_executor_instance; native_executor_instance!( pub Executor, node_runtime::api::dispatch, - node_runtime::native_version + node_runtime::native_version, + sp_io::benchmarking::HostFunctions, ); diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 4773a00d1d..763e8fc4fa 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -27,15 +27,18 @@ use frame_support::{ traits::{SplitTwoWays, Currency, Randomness}, }; use sp_core::u32_trait::{_1, _2, _3, _4}; -use node_primitives::{AccountId, AccountIndex, Balance, BlockNumber, Hash, Index, Moment, Signature}; +pub use node_primitives::{AccountId, Signature}; +use node_primitives::{AccountIndex, Balance, BlockNumber, Hash, Index, Moment}; use sp_api::impl_runtime_apis; use sp_runtime::{ - Permill, Perbill, Percent, ApplyExtrinsicResult, impl_opaque_keys, generic, create_runtime_str + Permill, Perbill, Percent, ApplyExtrinsicResult, BenchmarkResults, + impl_opaque_keys, generic, create_runtime_str, }; use sp_runtime::curve::PiecewiseLinear; use sp_runtime::transaction_validity::TransactionValidity; use sp_runtime::traits::{ - self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, ConvertInto, OpaqueKeys, + self, BlakeTwo256, Block as BlockT, StaticLookup, SaturatedConversion, + ConvertInto, OpaqueKeys, Benchmarking, }; use sp_version::RuntimeVersion; #[cfg(any(feature = "std", test))] @@ -79,8 +82,8 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 214, - impl_version: 3, + spec_version: 219, + impl_version: 0, apis: RUNTIME_API_VERSIONS, }; @@ -819,6 +822,27 @@ impl_runtime_apis! { SessionKeys::decode_into_raw_public_keys(&encoded) } } + + impl crate::Benchmark for Runtime { + fn dispatch_benchmark(module: Vec, extrinsic: Vec, steps: u32, repeat: u32) + -> Option> + { + match module.as_slice() { + b"pallet-balances" | b"balances" => Balances::run_benchmark(extrinsic, steps, repeat).ok(), + b"pallet-identity" | b"identity" => Identity::run_benchmark(extrinsic, steps, repeat).ok(), + b"pallet-timestamp" | b"timestamp" => Timestamp::run_benchmark(extrinsic, steps, repeat).ok(), + _ => return None, + } + } + } +} + +sp_api::decl_runtime_apis! { + pub trait Benchmark + { + fn dispatch_benchmark(module: Vec, extrinsic: Vec, steps: u32, repeat: u32) + -> Option>; + } } #[cfg(test)] diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index 051460e839..558c8f2f0b 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -9,6 +9,8 @@ license = "GPL-3.0" [dependencies] pallet-balances = { version = "2.0.0", path = "../../../frame/balances" } sc-client = { version = "0.8", path = "../../../client/" } +sc-client-db = { version = "0.8", path = "../../../client/db/", features = ["kvdb-rocksdb"] } +sc-client-api = { version = "2.0", path = "../../../client/api/" } codec = { package = "parity-scale-codec", version = "1.0.0" } pallet-contracts = { version = "2.0.0", path = "../../../frame/contracts" } pallet-grandpa = { version = "2.0.0", path = "../../../frame/grandpa" } @@ -24,10 +26,30 @@ pallet-session = { version = "2.0.0", path = "../../../frame/session" } pallet-society = { version = "2.0.0", path = "../../../frame/society" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } pallet-staking = { version = "2.0.0", path = "../../../frame/staking" } -sc-executor = { version = "0.8", path = "../../../client/executor" } +sc-executor = { version = "0.8", path = "../../../client/executor", features = ["wasmtime"] } +sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } frame-system = { version = "2.0.0", path = "../../../frame/system" } substrate-test-client = { version = "2.0.0", path = "../../../test-utils/client" } pallet-timestamp = { version = "2.0.0", path = "../../../frame/timestamp" } pallet-transaction-payment = { version = "2.0.0", path = "../../../frame/transaction-payment" } pallet-treasury = { version = "2.0.0", path = "../../../frame/treasury" } wabt = "0.9.2" +sp-api = { version = "2.0.0", path = "../../../primitives/api" } +sp-finality-tracker = { version = "2.0.0", default-features = false, path = "../../../primitives/finality-tracker" } +sp-timestamp = { version = "2.0.0", default-features = false, path = "../../../primitives/timestamp" } +sp-block-builder = { version = "2.0.0", path = "../../../primitives/block-builder" } +sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } +sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } +log = "0.4.8" + +[dev-dependencies] +criterion = "0.3.0" +tempdir = "0.3" +fs_extra = "1" +hex-literal = "0.2.1" +sc-cli = { version = "0.8.0", path = "../../../client/cli" } +sc-service = { version = "0.8.0", path = "../../../client/service", features = ["rocksdb"] } + +[[bench]] +name = "import" +harness = false diff --git a/bin/node/testing/benches/import.rs b/bin/node/testing/benches/import.rs new file mode 100644 index 0000000000..86092a7836 --- /dev/null +++ b/bin/node/testing/benches/import.rs @@ -0,0 +1,503 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Block import benchmark. +//! +//! This benchmark is expected to measure block import operation of +//! some more or less full block. +//! +//! As we also want to protect against cold-cache attacks, this +//! benchmark should not rely on any caching (except those that +//! DO NOT depend on user input). Thus block generation should be +//! based on randomized operation. +//! +//! This is supposed to be very simple benchmark and is not subject +//! to much configuring - just block full of randomized transactions. +//! It is not supposed to measure runtime modules weight correctness + +use std::{sync::Arc, path::Path, collections::BTreeMap}; + +use node_primitives::Block; +use node_testing::client::{Client, Backend}; +use node_testing::keyring::*; +use sc_client_db::PruningMode; +use sc_executor::{NativeExecutor, WasmExecutionMethod}; +use sp_consensus::{ + BlockOrigin, BlockImport, BlockImportParams, + ForkChoiceStrategy, ImportResult, ImportedAux +}; +use sp_runtime::{ + generic::BlockId, + OpaqueExtrinsic, + traits::{Block as BlockT, Verify, Zero, IdentifyAccount}, +}; +use codec::{Decode, Encode}; +use node_runtime::{ + Call, + CheckedExtrinsic, + constants::currency::DOLLARS, + UncheckedExtrinsic, + MinimumPeriod, + BalancesCall, + AccountId, + Signature, +}; +use sp_core::ExecutionContext; +use sp_api::ProvideRuntimeApi; +use sp_block_builder::BlockBuilder; +use sp_inherents::InherentData; +use sc_client_api::{ + Backend as _, ExecutionStrategy, + execution_extensions::{ExecutionExtensions, ExecutionStrategies}, +}; +use sp_core::{Pair, Public, sr25519}; + +use criterion::{Criterion, criterion_group, criterion_main}; + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = bench_block_import +); +criterion_group!( + name = profile; + config = Criterion::default().sample_size(10); + targets = profile_block_import +); +criterion_main!(benches, profile); + +fn genesis(keyring: &BenchKeyring) -> node_runtime::GenesisConfig { + node_testing::genesis::config_endowed( + false, + Some(node_runtime::WASM_BINARY), + keyring.collect_account_ids(), + ) +} + +// this is deterministic keyring of ordered accounts +// //endowed-user//00 +// //endowed-user//01 +// ... +// //endowed-user//N +struct BenchKeyring { + accounts: BTreeMap, +} + +impl BenchKeyring { + fn new(num: usize) -> Self { + let mut accounts = BTreeMap::new(); + + for n in 0..num { + let seed = format!("//endowed-user/{}", n); + let pair = sr25519::Pair::from_string(&seed, None).expect("failed to generate pair"); + let account_id = AccountPublic::from(pair.public()).into_account(); + accounts.insert(account_id, pair); + } + + Self { accounts } + } + + fn collect_account_ids(&self) -> Vec { + self.accounts.keys().cloned().collect() + } + + fn at(&self, index: usize) -> AccountId { + self.accounts.keys().nth(index).expect("Failed to get account").clone() + } + + fn sign(&self, xt: CheckedExtrinsic, version: u32, genesis_hash: [u8; 32]) -> UncheckedExtrinsic { + match xt.signed { + Some((signed, extra)) => { + let payload = (xt.function, extra.clone(), version, genesis_hash, genesis_hash); + let key = self.accounts.get(&signed).expect("Account id not found in keyring"); + let signature = payload.using_encoded(|b| { + if b.len() > 256 { + key.sign(&sp_io::hashing::blake2_256(b)) + } else { + key.sign(b) + } + }).into(); + UncheckedExtrinsic { + signature: Some((pallet_indices::address::Address::Id(signed), signature, extra)), + function: payload.0, + } + } + None => UncheckedExtrinsic { + signature: None, + function: xt.function, + }, + } + } +} + +#[derive(Clone, Copy, Debug)] +enum Profile { + Native, + Wasm, +} + +impl Profile { + fn into_execution_strategies(self) -> ExecutionStrategies { + match self { + Profile::Wasm => ExecutionStrategies { + syncing: ExecutionStrategy::AlwaysWasm, + importing: ExecutionStrategy::AlwaysWasm, + block_construction: ExecutionStrategy::AlwaysWasm, + offchain_worker: ExecutionStrategy::AlwaysWasm, + other: ExecutionStrategy::AlwaysWasm, + }, + Profile::Native => ExecutionStrategies { + syncing: ExecutionStrategy::NativeElseWasm, + importing: ExecutionStrategy::NativeElseWasm, + block_construction: ExecutionStrategy::NativeElseWasm, + offchain_worker: ExecutionStrategy::NativeElseWasm, + other: ExecutionStrategy::NativeElseWasm, + } + } + } +} + +// This should return client that is doing everything that full node +// is doing. +// +// - This client should use best wasm execution method. +// - This client should work with real database only. +fn bench_client(dir: &std::path::Path, profile: Profile, keyring: &BenchKeyring) -> (Client, std::sync::Arc) { + let db_config = sc_client_db::DatabaseSettings { + state_cache_size: 16*1024*1024, + state_cache_child_ratio: Some((0, 100)), + pruning: PruningMode::ArchiveAll, + source: sc_client_db::DatabaseSettingsSrc::Path { + path: dir.into(), + cache_size: None, + }, + }; + + let (client, backend) = sc_client_db::new_client( + db_config, + NativeExecutor::new(WasmExecutionMethod::Compiled, None), + &genesis(keyring), + None, + None, + ExecutionExtensions::new(profile.into_execution_strategies(), None), + ).expect("Should not fail"); + + (client, backend) +} + +struct Guard(tempdir::TempDir); + +struct BenchContext { + client: Client, + backend: Arc, + db_guard: Guard, + keyring: BenchKeyring, +} + +impl BenchContext { + fn new(profile: Profile) -> BenchContext { + let keyring = BenchKeyring::new(128); + + let dir = tempdir::TempDir::new("sub-bench").expect("temp dir creation failed"); + log::trace!( + target: "bench-logistics", + "Created seed db at {}", + dir.path().to_string_lossy(), + ); + let (client, backend) = bench_client(dir.path(), profile, &keyring); + let db_guard = Guard(dir); + + + BenchContext { client, backend, db_guard, keyring } + } + + fn new_from_seed(profile: Profile, seed_dir: &Path) -> BenchContext { + let keyring = BenchKeyring::new(128); + + let dir = tempdir::TempDir::new("sub-bench").expect("temp dir creation failed"); + + log::trace!( + target: "bench-logistics", + "Copying seed db from {} to {}", + seed_dir.to_string_lossy(), + dir.path().to_string_lossy(), + ); + let seed_db_files = std::fs::read_dir(seed_dir) + .expect("failed to list file in seed dir") + .map(|f_result| + f_result.expect("failed to read file in seed db") + .path() + .clone() + ).collect(); + fs_extra::copy_items( + &seed_db_files, + dir.path(), + &fs_extra::dir::CopyOptions::new(), + ).expect("Copy of seed database is ok"); + + let (client, backend) = bench_client(dir.path(), profile, &keyring); + let db_guard = Guard(dir); + + BenchContext { client, backend, db_guard, keyring } + } + + fn keep_db(self) -> Guard { + self.db_guard + } +} + +type AccountPublic = ::Signer; + +pub fn get_from_seed(seed: &str) -> ::Public { + TPublic::Pair::from_string(&format!("//{}", seed), None) + .expect("static values are valid; qed") + .public() +} + +pub fn get_account_id_from_seed(seed: &str) -> AccountId +where + AccountPublic: From<::Public> +{ + AccountPublic::from(get_from_seed::(seed)).into_account() +} + +// Block generation. +fn generate_block_import(client: &Client, keyring: &BenchKeyring) -> Block { + let version = client.runtime_version_at(&BlockId::number(0)) + .expect("There should be runtime version at 0") + .spec_version; + let genesis_hash = client.block_hash(Zero::zero()) + .expect("Database error?") + .expect("Genesis block always exists; qed") + .into(); + + let mut block = client + .new_block(Default::default()) + .expect("Block creation failed"); + + let timestamp = 1 * MinimumPeriod::get(); + + let mut inherent_data = InherentData::new(); + inherent_data.put_data(sp_timestamp::INHERENT_IDENTIFIER, ×tamp) + .expect("Put timestamb failed"); + inherent_data.put_data(sp_finality_tracker::INHERENT_IDENTIFIER, &0) + .expect("Put finality tracker failed"); + + for extrinsic in client.runtime_api() + .inherent_extrinsics_with_context( + &BlockId::number(0), + ExecutionContext::BlockConstruction, + inherent_data, + ).expect("Get inherents failed") + { + block.push(extrinsic).expect("Push inherent failed"); + } + + let mut iteration = 0; + let start = std::time::Instant::now(); + for _ in 0..100 { + + let sender = keyring.at(iteration); + let receiver = get_account_id_from_seed::( + &format!("random-user//{}", iteration) + ); + + let signed = keyring.sign( + CheckedExtrinsic { + signed: Some((sender, signed_extra(0, 1*DOLLARS))), + function: Call::Balances( + BalancesCall::transfer( + pallet_indices::address::Address::Id(receiver), + 1*DOLLARS + ) + ), + }, + version, + genesis_hash, + ); + + let encoded = Encode::encode(&signed); + + let opaque = OpaqueExtrinsic::decode(&mut &encoded[..]) + .expect("Failed to decode opaque"); + + match block.push(opaque) { + Err(sp_blockchain::Error::ApplyExtrinsicFailed( + sp_blockchain::ApplyExtrinsicFailed::Validity(e) + )) if e.exhausted_resources() => { + break; + }, + Err(err) => panic!("Error pushing transaction: {:?}", err), + Ok(_) => {}, + } + iteration += 1; + } + let block = block.build().expect("Block build failed").block; + + log::info!( + target: "bench-logistics", + "Block construction: {:#?} ({} tx)", + start.elapsed(), block.extrinsics.len() + ); + + block +} + +// Import generated block. +fn import_block(client: &mut Client, block: Block) { + let import_params = BlockImportParams { + origin: BlockOrigin::NetworkBroadcast, + header: block.header().clone(), + post_digests: Default::default(), + body: Some(block.extrinsics().to_vec()), + storage_changes: Default::default(), + finalized: false, + justification: Default::default(), + auxiliary: Default::default(), + intermediates: Default::default(), + fork_choice: Some(ForkChoiceStrategy::LongestChain), + allow_missing_state: false, + import_existing: false, + }; + + assert_eq!(client.chain_info().best_number, 0); + + assert_eq!( + client.import_block(import_params, Default::default()) + .expect("Failed to import block"), + ImportResult::Imported( + ImportedAux { + header_only: false, + clear_justification_requests: false, + needs_justification: false, + bad_justification: false, + needs_finality_proof: false, + is_new_best: true, + } + ) + ); + + assert_eq!(client.chain_info().best_number, 1); +} + +fn bench_block_import(c: &mut Criterion) { + sc_cli::init_logger(""); + // for future uses, uncomment if something wrong. + // sc_cli::init_logger("sc_client=debug"); + + let (block, guard) = { + let context = BenchContext::new(Profile::Wasm); + let block = generate_block_import(&context.client, &context.keyring); + (block, context.keep_db()) + }; + + log::trace!( + target: "bench-logistics", + "Seed database directory: {}", + guard.0.path().to_string_lossy(), + ); + + c.bench_function_over_inputs("import block", + move |bencher, profile| { + bencher.iter_batched( + || { + let context = BenchContext::new_from_seed( + *profile, + guard.0.path(), + ); + + // mostly to just launch compiler before benching! + let version = context.client.runtime_version_at(&BlockId::Number(0)) + .expect("Failed to get runtime version") + .spec_version; + + log::trace!( + target: "bench-logistics", + "Next iteration database directory: {}, runtime version: {}", + context.db_guard.0.path().to_string_lossy(), version, + ); + + context + }, + |mut context| { + let start = std::time::Instant::now(); + import_block(&mut context.client, block.clone()); + let elapsed = start.elapsed(); + + log::info!( + target: "bench-logistics", + "imported block with {} tx, took: {:#?}", + block.extrinsics.len(), + elapsed, + ); + + log::info!( + target: "bench-logistics", + "usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ); + }, + criterion::BatchSize::PerIteration, + ); + }, + vec![Profile::Wasm, Profile::Native], + ); +} + + +// This is not an actual benchmark, so don't use it to measure anything. +// It just produces special pattern of cpu load that allows easy picking +// the part of block import for the profiling in the tool of choice. +fn profile_block_import(c: &mut Criterion) { + sc_cli::init_logger(""); + + let (block, guard) = { + let context = BenchContext::new(Profile::Wasm); + let block = generate_block_import(&context.client, &context.keyring); + (block, context.keep_db()) + }; + + c.bench_function("profile block", + move |bencher| { + bencher.iter_batched( + || { + let context = BenchContext::new_from_seed( + Profile::Native, + guard.0.path(), + ); + context + }, + |mut context| { + // until better osx signpost/callgrind signal is possible to use + // in rust, we just pause everything completely to help choosing + // actual profiling interval + std::thread::park_timeout(std::time::Duration::from_secs(2)); + import_block(&mut context.client, block.clone()); + // and here as well + std::thread::park_timeout(std::time::Duration::from_secs(2)); + log::info!( + target: "bench-logistics", + "imported block, usage info: {}", + context.backend.usage_info() + .expect("RocksDB backend always provides usage info!"), + ) + }, + criterion::BatchSize::PerIteration, + ); + }, + ); +} \ No newline at end of file diff --git a/bin/node/testing/src/client.rs b/bin/node/testing/src/client.rs index 1dddd8ba5a..29b086c3c1 100644 --- a/bin/node/testing/src/client.rs +++ b/bin/node/testing/src/client.rs @@ -35,6 +35,9 @@ pub type Client = sc_client::Client< node_runtime::RuntimeApi, >; +/// Transaction for node-runtime. +pub type Transaction = sc_client_api::backend::TransactionFor; + /// Genesis configuration parameters for `TestClient`. #[derive(Default)] pub struct GenesisParameters { diff --git a/bin/node/testing/src/genesis.rs b/bin/node/testing/src/genesis.rs index 8c5514defa..58e646e3d7 100644 --- a/bin/node/testing/src/genesis.rs +++ b/bin/node/testing/src/genesis.rs @@ -21,14 +21,38 @@ use sp_keyring::{Ed25519Keyring, Sr25519Keyring}; use node_runtime::{ GenesisConfig, BalancesConfig, SessionConfig, StakingConfig, SystemConfig, GrandpaConfig, IndicesConfig, ContractsConfig, SocietyConfig, WASM_BINARY, + AccountId, }; use node_runtime::constants::currency::*; use sp_core::ChangesTrieConfiguration; use sp_runtime::Perbill; - /// Create genesis runtime configuration for tests. pub fn config(support_changes_trie: bool, code: Option<&[u8]>) -> GenesisConfig { + config_endowed(support_changes_trie, code, Default::default()) +} + +/// Create genesis runtime configuration for tests with some extra +/// endowed accounts. +pub fn config_endowed( + support_changes_trie: bool, + code: Option<&[u8]>, + extra_endowed: Vec, +) -> GenesisConfig { + + let mut endowed = vec![ + (alice(), 111 * DOLLARS), + (bob(), 100 * DOLLARS), + (charlie(), 100_000_000 * DOLLARS), + (dave(), 111 * DOLLARS), + (eve(), 101 * DOLLARS), + (ferdie(), 100 * DOLLARS), + ]; + + endowed.extend( + extra_endowed.into_iter().map(|endowed| (endowed, 100*DOLLARS)) + ); + GenesisConfig { frame_system: Some(SystemConfig { changes_trie_config: if support_changes_trie { Some(ChangesTrieConfiguration { @@ -41,14 +65,7 @@ pub fn config(support_changes_trie: bool, code: Option<&[u8]>) -> GenesisConfig ids: vec![alice(), bob(), charlie(), dave(), eve(), ferdie()], }), pallet_balances: Some(BalancesConfig { - balances: vec![ - (alice(), 111 * DOLLARS), - (bob(), 100 * DOLLARS), - (charlie(), 100_000_000 * DOLLARS), - (dave(), 111 * DOLLARS), - (eve(), 101 * DOLLARS), - (ferdie(), 100 * DOLLARS), - ], + balances: endowed, }), pallet_session: Some(SessionConfig { keys: vec![ diff --git a/bin/node/transaction-factory/src/complex_mode.rs b/bin/node/transaction-factory/src/complex_mode.rs deleted file mode 100644 index 6d7e60c8d3..0000000000 --- a/bin/node/transaction-factory/src/complex_mode.rs +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -/// This module implements the `MasterToNToM` mode: -/// -/// Manufacture `num` transactions from the master account to `num` -/// randomly created accounts. From each of these randomly created -/// accounts manufacture a transaction to another randomly created -/// account. -/// Repeat this round `rounds` times. If `rounds` = 1 the behavior -/// is the same as `MasterToN`. -/// -/// A -> B -/// A -> C -/// A -> D -/// ... x `num` -/// -/// B -> E -/// C -> F -/// D -> G -/// ... -/// E -> H -/// F -> I -/// G -> J -/// ... -/// ... x `rounds` - -use std::sync::Arc; - -use log::info; -use sc_client::Client; -use sp_block_builder::BlockBuilder; -use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi}; -use sp_runtime::generic::BlockId; -use sp_runtime::traits::{Block as BlockT, One, Zero}; - -use crate::{RuntimeAdapter, create_block}; - -pub fn next( - factory_state: &mut RA, - client: &Arc>, - version: u32, - genesis_hash: ::Hash, - prior_block_hash: ::Hash, - prior_block_id: BlockId, -) -> Option -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilder + - sp_api::ApiExt, - RtApi: ConstructRuntimeApi> + Send + Sync, - RA: RuntimeAdapter, -{ - let total = factory_state.start_number() + factory_state.num() * factory_state.rounds(); - - if factory_state.block_no() >= total || factory_state.round() >= factory_state.rounds() { - return None; - } - - info!( - "Round {}: Creating {} transactions in total, {} per round. {} rounds in total.", - factory_state.round() + RA::Number::one(), - factory_state.num() * factory_state.rounds(), - factory_state.num(), - factory_state.rounds(), - ); - - let from = from::(factory_state); - - let seed = factory_state.start_number() + factory_state.block_no(); - let to = RA::gen_random_account_id(&seed); - - let rounds_left = factory_state.rounds() - factory_state.round(); - let amount = RA::minimum_balance() * rounds_left.into(); - - let transfer = factory_state.transfer_extrinsic( - &from.0, - &from.1, - &to, - &amount, - version, - &genesis_hash, - &prior_block_hash, - ); - - let inherents = factory_state.inherent_extrinsics(); - let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents) - .expect("Failed to create inherent extrinsics"); - - let block = create_block::(&client, transfer, inherents); - info!( - "Created block {} with hash {}. Transferring {} from {} to {}.", - factory_state.block_no() + RA::Number::one(), - prior_block_hash, - amount, - from.0, - to - ); - - factory_state.set_block_no(factory_state.block_no() + RA::Number::one()); - - let new_round = factory_state.block_no() > RA::Number::zero() - && factory_state.block_no() % factory_state.num() == RA::Number::zero(); - if new_round { - factory_state.set_round(factory_state.round() + RA::Number::one()); - factory_state.set_block_in_round(RA::Number::zero()); - } else { - factory_state.set_block_in_round(factory_state.block_in_round() + RA::Number::one()); - } - - Some(block) -} - -/// Return the account which received tokens at this point in the previous round. -fn from( - factory_state: &mut RA -) -> (::AccountId, ::Secret) -where RA: RuntimeAdapter -{ - let is_first_round = factory_state.round() == RA::Number::zero(); - match is_first_round { - true => { - // first round always uses master account - (RA::master_account_id(), RA::master_account_secret()) - }, - _ => { - // the account to which was sent in the last round - let is_round_one = factory_state.round() == RA::Number::one(); - let seed = match is_round_one { - true => factory_state.start_number() + factory_state.block_in_round(), - _ => { - let block_no_in_prior_round = - factory_state.num() * (factory_state.round() - RA::Number::one()) + factory_state.block_in_round(); - factory_state.start_number() + block_no_in_prior_round - } - }; - (RA::gen_random_account_id(&seed), RA::gen_random_account_secret(&seed)) - }, - } -} diff --git a/bin/node/transaction-factory/src/lib.rs b/bin/node/transaction-factory/src/lib.rs index a0c6a4f663..acee44625f 100644 --- a/bin/node/transaction-factory/src/lib.rs +++ b/bin/node/transaction-factory/src/lib.rs @@ -37,39 +37,28 @@ use sp_consensus::block_import::BlockImport; use codec::{Decode, Encode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{ - Block as BlockT, Header as HeaderT, SimpleArithmetic, One, Zero, + Block as BlockT, Header as HeaderT, AtLeast32Bit, One, Zero, }; -pub use crate::modes::Mode; - -pub mod modes; -mod complex_mode; -mod simple_modes; pub trait RuntimeAdapter { type AccountId: Display; - type Balance: Display + SimpleArithmetic + From; + type Balance: Display + AtLeast32Bit + From; type Block: BlockT; type Index: Copy; - type Number: Display + PartialOrd + SimpleArithmetic + Zero + One; + type Number: Display + PartialOrd + AtLeast32Bit + Zero + One; type Phase: Copy; type Secret; - fn new(mode: Mode, rounds: u64, start_number: u64) -> Self; + fn new(blocks: u32, transactions: u32) -> Self; - fn block_no(&self) -> Self::Number; - fn block_in_round(&self) -> Self::Number; - fn mode(&self) -> &Mode; - fn num(&self) -> Self::Number; - fn rounds(&self) -> Self::Number; - fn round(&self) -> Self::Number; - fn start_number(&self) -> Self::Number; + fn blocks(&self) -> u32; + fn transactions(&self) -> u32; - fn set_block_in_round(&mut self, val: Self::Number); - fn set_block_no(&mut self, val: Self::Number); - fn set_round(&mut self, val: Self::Number); + fn block_number(&self) -> u32; + fn set_block_number(&mut self, value: u32); fn transfer_extrinsic( - &self, + &mut self, sender: &Self::AccountId, key: &Self::Secret, destination: &Self::AccountId, @@ -84,14 +73,12 @@ pub trait RuntimeAdapter { fn minimum_balance() -> Self::Balance; fn master_account_id() -> Self::AccountId; fn master_account_secret() -> Self::Secret; - fn extract_index(&self, account_id: &Self::AccountId, block_hash: &::Hash) -> Self::Index; - fn extract_phase(&self, block_hash: ::Hash) -> Self::Phase; - fn gen_random_account_id(seed: &Self::Number) -> Self::AccountId; - fn gen_random_account_secret(seed: &Self::Number) -> Self::Secret; + + fn gen_random_account_id(seed: u32) -> Self::AccountId; + fn gen_random_account_secret(seed: u32) -> Self::Secret; } -/// Manufactures transactions. The exact amount depends on -/// `mode`, `num` and `rounds`. +/// Manufactures transactions. The exact amount depends on `num` and `rounds`. pub fn factory( mut factory_state: RA, client: &Arc>, @@ -110,11 +97,6 @@ where RA: RuntimeAdapter, Block::Hash: From, { - if *factory_state.mode() != Mode::MasterToNToM && factory_state.rounds() > RA::Number::one() { - let msg = "The factory can only be used with rounds set to 1 in this mode.".into(); - return Err(sc_cli::error::Error::Input(msg)); - } - let best_header: Result<::Header, sc_cli::error::Error> = select_chain.best_chain().map_err(|e| format!("{:?}", e).into()); let mut best_hash = best_header?.hash(); @@ -123,89 +105,75 @@ where let genesis_hash = client.block_hash(Zero::zero())? .expect("Genesis block always exists; qed").into(); - while let Some(block) = match factory_state.mode() { - Mode::MasterToNToM => complex_mode::next::( - &mut factory_state, - &client, - version, - genesis_hash, - best_hash.into(), - best_block_id, - ), - _ => simple_modes::next::( - &mut factory_state, - &client, - version, - genesis_hash, - best_hash.into(), - best_block_id, - ), - } { + while factory_state.block_number() < factory_state.blocks() { + let from = (RA::master_account_id(), RA::master_account_secret()); + let amount = RA::minimum_balance(); + + let inherents = RA::inherent_extrinsics(&factory_state); + let inherents = client.runtime_api().inherent_extrinsics(&best_block_id, inherents) + .expect("Failed to create inherent extrinsics"); + + let tx_per_block = factory_state.transactions(); + + let mut block = client.new_block(Default::default()).expect("Failed to create new block"); + + for tx_num in 0..tx_per_block { + let seed = tx_num * (factory_state.block_number() + 1); + let to = RA::gen_random_account_id(seed); + + let transfer = factory_state.transfer_extrinsic( + &from.0, + &from.1, + &to, + &amount, + version, + &genesis_hash, + &best_hash, + ); + + info!("Pushing transfer {}/{} to {} into block.", tx_num + 1, tx_per_block, to); + + block.push( + Decode::decode(&mut &transfer.encode()[..]) + .expect("Failed to decode transfer extrinsic") + ).expect("Failed to push transfer extrinsic into block"); + } + + for inherent in inherents { + block.push(inherent).expect("Failed ..."); + } + + let block = block.build().expect("Failed to bake block").block; + + factory_state.set_block_number(factory_state.block_number() + 1); + + info!( + "Created block {} with hash {}.", + factory_state.block_number(), + best_hash, + ); + best_hash = block.header().hash(); best_block_id = BlockId::::hash(best_hash); - import_block(client.clone(), block); - info!("Imported block at {}", factory_state.block_no()); + let import = BlockImportParams { + origin: BlockOrigin::File, + header: block.header().clone(), + post_digests: Vec::new(), + body: Some(block.extrinsics().to_vec()), + storage_changes: None, + finalized: false, + justification: None, + auxiliary: Vec::new(), + intermediates: Default::default(), + fork_choice: Some(ForkChoiceStrategy::LongestChain), + allow_missing_state: false, + import_existing: false, + }; + client.clone().import_block(import, HashMap::new()).expect("Failed to import block"); + + info!("Imported block at {}", factory_state.block_number()); } Ok(()) } - -/// Create a baked block from a transfer extrinsic and timestamp inherent. -pub fn create_block( - client: &Arc>, - transfer: ::Extrinsic, - inherent_extrinsics: Vec<::Extrinsic>, -) -> Block -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - RtApi: ConstructRuntimeApi> + Send + Sync, - as ProvideRuntimeApi>::Api: - BlockBuilder + - ApiExt, - RA: RuntimeAdapter, -{ - let mut block = client.new_block(Default::default()).expect("Failed to create new block"); - block.push( - Decode::decode(&mut &transfer.encode()[..]) - .expect("Failed to decode transfer extrinsic") - ).expect("Failed to push transfer extrinsic into block"); - - for inherent in inherent_extrinsics { - block.push(inherent).expect("Failed ..."); - } - - block.build().expect("Failed to bake block").block -} - -fn import_block( - mut client: Arc>, - block: Block -) -> () where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - sp_api::Core + - ApiExt, -{ - let import = BlockImportParams { - origin: BlockOrigin::File, - header: block.header().clone(), - post_digests: Vec::new(), - body: Some(block.extrinsics().to_vec()), - storage_changes: None, - finalized: false, - justification: None, - auxiliary: Vec::new(), - intermediates: Default::default(), - fork_choice: Some(ForkChoiceStrategy::LongestChain), - allow_missing_state: false, - import_existing: false, - }; - client.import_block(import, HashMap::new()).expect("Failed to import block"); -} diff --git a/bin/node/transaction-factory/src/modes.rs b/bin/node/transaction-factory/src/modes.rs deleted file mode 100644 index 5deab7635e..0000000000 --- a/bin/node/transaction-factory/src/modes.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! The transaction factory can operate in different modes. See -//! the `simple_mode` and `complex_mode` modules for details. - -use std::str::FromStr; - -/// Token distribution modes. -#[derive(Debug, Clone, PartialEq)] -pub enum Mode { - MasterToN, - MasterTo1, - MasterToNToM -} - -impl FromStr for Mode { - type Err = String; - fn from_str(mode: &str) -> Result { - match mode { - "MasterToN" => Ok(Mode::MasterToN), - "MasterTo1" => Ok(Mode::MasterTo1), - "MasterToNToM" => Ok(Mode::MasterToNToM), - _ => Err(format!("Invalid mode: {}", mode)), - } - } -} diff --git a/bin/node/transaction-factory/src/simple_modes.rs b/bin/node/transaction-factory/src/simple_modes.rs deleted file mode 100644 index fba328731a..0000000000 --- a/bin/node/transaction-factory/src/simple_modes.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -/// This module implements two manufacturing modes: -/// -/// # MasterToN -/// Manufacture `num` transactions from the master account -/// to `num` randomly created accounts, one each. -/// -/// A -> B -/// A -> C -/// ... x `num` -/// -/// -/// # MasterTo1 -/// Manufacture `num` transactions from the master account -/// to exactly one other randomly created account. -/// -/// A -> B -/// A -> B -/// ... x `num` - -use std::sync::Arc; - -use log::info; -use sc_client::Client; -use sp_block_builder::BlockBuilder; -use sp_api::{ConstructRuntimeApi, ProvideRuntimeApi}; -use sp_runtime::traits::{Block as BlockT, One}; -use sp_runtime::generic::BlockId; - -use crate::{Mode, RuntimeAdapter, create_block}; - -pub fn next( - factory_state: &mut RA, - client: &Arc>, - version: u32, - genesis_hash: ::Hash, - prior_block_hash: ::Hash, - prior_block_id: BlockId, -) -> Option -where - Block: BlockT, - Exec: sc_client::CallExecutor + Send + Sync + Clone, - Backend: sc_client_api::backend::Backend + Send, - Client: ProvideRuntimeApi, - as ProvideRuntimeApi>::Api: - BlockBuilder + - sp_api::ApiExt, - RtApi: ConstructRuntimeApi> + Send + Sync, - RA: RuntimeAdapter, -{ - if factory_state.block_no() >= factory_state.num() { - return None; - } - - let from = (RA::master_account_id(), RA::master_account_secret()); - - let seed = match factory_state.mode() { - // choose the same receiver for all transactions - Mode::MasterTo1 => factory_state.start_number(), - - // different receiver for each transaction - Mode::MasterToN => factory_state.start_number() + factory_state.block_no(), - _ => unreachable!("Mode not covered!"), - }; - let to = RA::gen_random_account_id(&seed); - - let amount = RA::minimum_balance(); - - let transfer = factory_state.transfer_extrinsic( - &from.0, - &from.1, - &to, - &amount, - version, - &genesis_hash, - &prior_block_hash, - ); - - let inherents = RA::inherent_extrinsics(&factory_state); - let inherents = client.runtime_api().inherent_extrinsics(&prior_block_id, inherents) - .expect("Failed to create inherent extrinsics"); - - let block = create_block::(&client, transfer, inherents); - - factory_state.set_block_no(factory_state.block_no() + RA::Number::one()); - - info!( - "Created block {} with hash {}. Transferring {} from {} to {}.", - factory_state.block_no(), - prior_block_hash, - amount, - from.0, - to - ); - - Some(block) -} diff --git a/bin/utils/subkey/Cargo.toml b/bin/utils/subkey/Cargo.toml index 9b7db6699f..e29ee9a6c5 100644 --- a/bin/utils/subkey/Cargo.toml +++ b/bin/utils/subkey/Cargo.toml @@ -28,6 +28,8 @@ derive_more = { version = "0.99.2" } sc-rpc = { version = "2.0.0", path = "../../../client/rpc" } jsonrpc-core-client = { version = "14.0.3", features = ["http"] } hyper = "0.12.35" +libp2p = "0.15.0" +serde_json = "1.0" [features] bench = [] diff --git a/bin/utils/subkey/src/main.rs b/bin/utils/subkey/src/main.rs index 9b70a8ace8..996fe91562 100644 --- a/bin/utils/subkey/src/main.rs +++ b/bin/utils/subkey/src/main.rs @@ -23,8 +23,10 @@ use clap::{App, ArgMatches, SubCommand}; use codec::{Decode, Encode}; use hex_literal::hex; use itertools::Itertools; +use libp2p::identity::{ed25519 as libp2p_ed25519, PublicKey}; use node_primitives::{Balance, Hash, Index, AccountId, Signature}; use node_runtime::{BalancesCall, Call, Runtime, SignedPayload, UncheckedExtrinsic, VERSION}; +use serde_json::json; use sp_core::{ crypto::{set_default_ss58_version, Ss58AddressFormat, Ss58Codec}, ed25519, sr25519, ecdsa, Pair, Public, H256, hexdisplay::HexDisplay, @@ -37,6 +39,24 @@ use std::{ mod rpc; mod vanity; +enum OutputType { + Json, + Text, +} + +impl<'a> TryFrom<&'a str> for OutputType { + type Error = (); + + fn try_from(s: &'a str) -> Result { + match s { + "json" => Ok(OutputType::Json), + "text" => Ok(OutputType::Text), + _ => Err(()), + } + } + +} + trait Crypto: Sized { type Pair: Pair; type Public: Public + Ss58Codec + AsRef<[u8]> + std::hash::Hash; @@ -55,50 +75,97 @@ trait Crypto: Sized { uri: &str, password: Option<&str>, network_override: Option, + output: OutputType, ) where ::Public: PublicT, { if let Ok((pair, seed)) = Self::Pair::from_phrase(uri, password) { let public_key = Self::public_from_pair(&pair); - println!("Secret phrase `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - format_seed::(seed), - format_public_key::(public_key.clone()), - format_account_id::(public_key), - Self::ss58_from_pair(&pair) - ); + + match output { + OutputType::Json => { + let json = json!({ + "secretPhrase": uri, + "secretSeed": format_seed::(seed), + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key), + "ss58Address": Self::ss58_from_pair(&pair), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Secret phrase `{}` is account:\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + format_seed::(seed), + format_public_key::(public_key.clone()), + format_account_id::(public_key), + Self::ss58_from_pair(&pair), + ); + }, + } } else if let Ok((pair, seed)) = Self::Pair::from_string_with_seed(uri, password) { let public_key = Self::public_from_pair(&pair); - println!("Secret Key URI `{}` is account:\n \ - Secret seed: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, - format_public_key::(public_key.clone()), - format_account_id::(public_key), - Self::ss58_from_pair(&pair) - ); + + match output { + OutputType::Json => { + let json = json!({ + "secretKeyUri": uri, + "secretSeed": if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key), + "ss58Address": Self::ss58_from_pair(&pair), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Secret Key URI `{}` is account:\n \ + Secret seed: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + if let Some(seed) = seed { format_seed::(seed) } else { "n/a".into() }, + format_public_key::(public_key.clone()), + format_account_id::(public_key), + Self::ss58_from_pair(&pair), + ); + }, + } + } else if let Ok((public_key, v)) = ::Public::from_string_with_version(uri) { let v = network_override.unwrap_or(v); - println!("Public Key URI `{}` is account:\n \ - Network ID/version: {}\n \ - Public key (hex): {}\n \ - Account ID: {}\n \ - SS58 Address: {}", - uri, - String::from(v), - format_public_key::(public_key.clone()), - format_account_id::(public_key.clone()), - public_key.to_ss58check_with_version(v) - ); + + match output { + OutputType::Json => { + let json = json!({ + "publicKeyUri": uri, + "networkId": String::from(v), + "publicKey": format_public_key::(public_key.clone()), + "accountId": format_account_id::(public_key.clone()), + "ss58Address": public_key.to_ss58check_with_version(v), + }); + println!("{}", serde_json::to_string_pretty(&json).expect("Json pretty print failed")); + }, + OutputType::Text => { + println!("Public Key URI `{}` is account:\n \ + Network ID/version: {}\n \ + Public key (hex): {}\n \ + Account ID: {}\n \ + SS58 Address: {}", + uri, + String::from(v), + format_public_key::(public_key.clone()), + format_account_id::(public_key.clone()), + public_key.to_ss58check_with_version(v), + ); + }, + } } else { println!("Invalid phrase/URI given"); } @@ -165,6 +232,7 @@ fn get_usage() -> String { [network] -n, --network 'Specify a network. One of {}. Default is {}' [password] -p, --password 'The password for the key' --password-interactive 'You will be prompted for the password for the key.' + [output] -o, --output 'Specify an output format. One of text, json. Default is text.' ", networks, default_network) } @@ -181,6 +249,9 @@ fn get_app<'a, 'b>(usage: &'a str) -> App<'a, 'b> { 'The number of words in the phrase to generate. One of 12 \ (default), 15, 18, 21 and 24.' "), + SubCommand::with_name("generate-node-key") + .about("Generate a random node libp2p key, save it to file and print its peer ID") + .args_from_usage("[file] 'Name of file to save secret key to'"), SubCommand::with_name("inspect") .about("Gets a public key and a SS58 address from the provided Secret URI") .args_from_usage("[uri] 'A Key URI to be inspected. May be a secret seed, \ @@ -329,13 +400,31 @@ where if let Some(network) = maybe_network { set_default_ss58_version(network); } + + let output: OutputType = match matches.value_of("output").map(TryInto::try_into) { + Some(Err(_)) => return Err(Error::Static("Invalid output name. See --help for available outputs.")), + Some(Ok(v)) => v, + None => OutputType::Text, + }; + match matches.subcommand() { ("generate", Some(matches)) => { let mnemonic = generate_mnemonic(matches)?; - C::print_from_uri(mnemonic.phrase(), password, maybe_network); + C::print_from_uri(mnemonic.phrase(), password, maybe_network, output); + } + ("generate-node-key", Some(matches)) => { + let file = matches.value_of("file").ok_or(Error::Static("Output file name is required"))?; + + let keypair = libp2p_ed25519::Keypair::generate(); + let secret = keypair.secret(); + let peer_id = PublicKey::Ed25519(keypair.public()).into_peer_id(); + + fs::write(file, secret.as_ref())?; + + println!("{}", peer_id); } ("inspect", Some(matches)) => { - C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network); + C::print_from_uri(&get_uri("uri", &matches)?, password, maybe_network, output); } ("sign", Some(matches)) => { let suri = get_uri("suri", &matches)?; @@ -364,7 +453,7 @@ where .unwrap_or_default(); let result = vanity::generate_key::(&desired)?; let formated_seed = format_seed::(result.seed); - C::print_from_uri(&formated_seed, None, maybe_network); + C::print_from_uri(&formated_seed, None, maybe_network, output); } ("transfer", Some(matches)) => { let signer = read_pair::(matches.value_of("from"), password)?; diff --git a/client/api/src/client.rs b/client/api/src/client.rs index 65952ce787..7503ce4a79 100644 --- a/client/api/src/client.rs +++ b/client/api/src/client.rs @@ -125,6 +125,8 @@ pub struct IoInfo { pub state_reads: u64, /// State reads (keys) from cache. pub state_reads_cache: u64, + /// State reads (keys) from cache. + pub state_writes: u64, } /// Usage statistics for running client instance. @@ -143,7 +145,7 @@ pub struct UsageInfo { impl fmt::Display for UsageInfo { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, - "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total)", + "caches: ({} state, {} db overlay), i/o: ({} tx, {} write, {} read, {} avg tx, {}/{} key cache reads/total, {} key writes)", self.memory.state_cache, self.memory.database_cache, self.io.transactions, @@ -152,6 +154,7 @@ impl fmt::Display for UsageInfo { self.io.average_transaction_size, self.io.state_reads_cache, self.io.state_reads, + self.io.state_writes, ) } } diff --git a/client/authority-discovery/Cargo.toml b/client/authority-discovery/Cargo.toml index 68144625b3..40b1b30173 100644 --- a/client/authority-discovery/Cargo.toml +++ b/client/authority-discovery/Cargo.toml @@ -14,7 +14,7 @@ bytes = "0.4.12" codec = { package = "parity-scale-codec", default-features = false, version = "1.0.3" } derive_more = "0.99.2" futures = "0.3.1" -futures-timer = "2.0" +futures-timer = "3.0.1" libp2p = { version = "0.15.0", default-features = false, features = ["secp256k1", "libp2p-websocket"] } log = "0.4.8" prost = "0.6.1" diff --git a/client/cli/Cargo.toml b/client/cli/Cargo.toml index e302d53d55..e176894d64 100644 --- a/client/cli/Cargo.toml +++ b/client/cli/Cargo.toml @@ -21,6 +21,7 @@ tokio = { version = "0.2.9", features = [ "signal", "rt-core", "rt-threaded" ] } futures = "0.3.1" fdlimit = "0.1.1" serde_json = "1.0.41" +sc-informant = { version = "0.8", path = "../informant" } sp-panic-handler = { version = "2.0.0", path = "../../primitives/panic-handler" } sc-client-api = { version = "2.0.0", path = "../api" } sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } diff --git a/client/cli/src/lib.rs b/client/cli/src/lib.rs index 54a98a6ad3..7495ad8e75 100644 --- a/client/cli/src/lib.rs +++ b/client/cli/src/lib.rs @@ -24,7 +24,6 @@ mod traits; mod params; mod execution_strategy; pub mod error; -pub mod informant; mod runtime; mod node_key; @@ -58,6 +57,7 @@ use params::{ pub use params::{ SharedParams, ImportParams, ExecutionStrategy, Subcommand, RunCmd, BuildSpecCmd, ExportBlocksCmd, ImportBlocksCmd, CheckBlockCmd, PurgeChainCmd, RevertCmd, + BenchmarkCmd, }; pub use traits::GetSharedParams; use app_dirs::{AppInfo, AppDataType}; @@ -424,6 +424,7 @@ fn fill_network_configuration( enable_mdns: !is_dev && !cli.no_mdns, allow_private_ipv4: !cli.no_private_ipv4, wasm_external_transport: None, + use_yamux_flow_control: cli.use_yamux_flow_control }; config.max_parallel_downloads = cli.max_parallel_downloads; diff --git a/client/cli/src/params.rs b/client/cli/src/params.rs index a99d887d25..2ffa8bd61b 100644 --- a/client/cli/src/params.rs +++ b/client/cli/src/params.rs @@ -238,6 +238,10 @@ pub struct NetworkConfigurationParams { #[allow(missing_docs)] #[structopt(flatten)] pub node_key_params: NodeKeyParams, + + /// Experimental feature flag. + #[structopt(long = "use-yamux-flow-control")] + pub use_yamux_flow_control: bool, } arg_enum! { @@ -845,6 +849,49 @@ pub struct PurgeChainCmd { pub shared_params: SharedParams, } +/// The `benchmark` command used to benchmark FRAME Pallets. +#[derive(Debug, StructOpt, Clone)] +pub struct BenchmarkCmd { + /// Select a FRAME Pallet to benchmark. + #[structopt(short, long)] + pub pallet: String, + + /// Select an extrinsic to benchmark. + #[structopt(short, long)] + pub extrinsic: String, + + /// Select how many samples we should take across the variable components. + #[structopt(short, long, default_value = "1")] + pub steps: u32, + + /// Select how many repetitions of this benchmark should run. + #[structopt(short, long, default_value = "1")] + pub repeat: u32, + + #[allow(missing_docs)] + #[structopt(flatten)] + pub shared_params: SharedParams, + + /// The execution strategy that should be used for benchmarks + #[structopt( + long = "execution", + value_name = "STRATEGY", + possible_values = &ExecutionStrategy::variants(), + case_insensitive = true, + )] + pub execution: Option, + + /// Method for executing Wasm runtime code. + #[structopt( + long = "wasm-execution", + value_name = "METHOD", + possible_values = &WasmExecutionMethod::enabled_variants(), + case_insensitive = true, + default_value = "Interpreted" + )] + pub wasm_method: WasmExecutionMethod, +} + /// All core commands that are provided by default. /// /// The core commands are split into multiple subcommands and `Run` is the default subcommand. From @@ -869,6 +916,9 @@ pub enum Subcommand { /// Remove the whole chain data. PurgeChain(PurgeChainCmd), + + /// Run runtime benchmarks. + Benchmark(BenchmarkCmd), } impl Subcommand { @@ -883,6 +933,7 @@ impl Subcommand { CheckBlock(params) => ¶ms.shared_params, Revert(params) => ¶ms.shared_params, PurgeChain(params) => ¶ms.shared_params, + Benchmark(params) => ¶ms.shared_params, } } @@ -909,6 +960,7 @@ impl Subcommand { Subcommand::ImportBlocks(cmd) => cmd.run(config, builder), Subcommand::CheckBlock(cmd) => cmd.run(config, builder), Subcommand::PurgeChain(cmd) => cmd.run(config), + Subcommand::Benchmark(cmd) => cmd.run(config, builder), Subcommand::Revert(cmd) => cmd.run(config, builder), } } @@ -1186,3 +1238,32 @@ impl RevertCmd { Ok(()) } } + +impl BenchmarkCmd { + /// Runs the command and benchmarks the chain. + pub fn run( + self, + config: Configuration, + _builder: B, + ) -> error::Result<()> + where + B: FnOnce(Configuration) -> Result, + G: RuntimeGenesis, + E: ChainSpecExtension, + BC: ServiceBuilderCommand + Unpin, + BB: sp_runtime::traits::Block + Debug, + <<::Header as HeaderT>::Number as std::str::FromStr>::Err: std::fmt::Debug, + ::Hash: std::str::FromStr, + { + let spec = config.chain_spec.expect("chain_spec is always Some"); + let execution_strategy = self.execution.unwrap_or(ExecutionStrategy::Native).into(); + let wasm_method = self.wasm_method.into(); + let pallet = self.pallet; + let extrinsic = self.extrinsic; + let steps = self.steps; + let repeat = self.repeat; + sc_service::chain_ops::benchmark_runtime::(spec, execution_strategy, wasm_method, pallet, extrinsic, steps, repeat)?; + Ok(()) + } +} + diff --git a/client/cli/src/runtime.rs b/client/cli/src/runtime.rs index 62a2245c9e..157b75f205 100644 --- a/client/cli/src/runtime.rs +++ b/client/cli/src/runtime.rs @@ -21,7 +21,6 @@ use futures::select; use futures::pin_mut; use sc_service::{AbstractService, Configuration}; use crate::error; -use crate::informant; #[cfg(target_family = "unix")] async fn main(func: F) -> Result<(), Box> @@ -124,7 +123,7 @@ where let service = service_builder(config)?; - let informant_future = informant::build(&service); + let informant_future = sc_informant::build(&service, sc_informant::OutputFormat::Coloured); let _informant_handle = runtime.spawn(informant_future); // we eagerly drop the service so that the internal exit future is fired, diff --git a/client/consensus/aura/Cargo.toml b/client/consensus/aura/Cargo.toml index 142b10b9da..e67f1e15a3 100644 --- a/client/consensus/aura/Cargo.toml +++ b/client/consensus/aura/Cargo.toml @@ -16,7 +16,7 @@ codec = { package = "parity-scale-codec", version = "1.0.0" } sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } derive_more = "0.99.2" futures = "0.3.1" -futures-timer = "0.4.0" +futures-timer = "3.0.1" sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } sc-keystore = { version = "2.0.0", path = "../../keystore" } log = "0.4.8" diff --git a/client/consensus/babe/Cargo.toml b/client/consensus/babe/Cargo.toml index 23bf9cea28..c36b5216c2 100644 --- a/client/consensus/babe/Cargo.toml +++ b/client/consensus/babe/Cargo.toml @@ -32,7 +32,7 @@ sc-consensus-slots = { version = "0.8", path = "../slots" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } fork-tree = { version = "2.0.0", path = "../../../utils/fork-tree" } futures = "0.3.1" -futures-timer = "0.4.0" +futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" schnorrkel = { version = "0.8.5", features = ["preaudit_deprecated"] } diff --git a/client/consensus/manual-seal/src/lib.rs b/client/consensus/manual-seal/src/lib.rs index 029f746e62..c820407e74 100644 --- a/client/consensus/manual-seal/src/lib.rs +++ b/client/consensus/manual-seal/src/lib.rs @@ -198,7 +198,7 @@ pub async fn run_instant_seal( { // instant-seal creates blocks as soon as transactions are imported // into the transaction pool. - let seal_block_channel = pool.import_notification_stream() + let seal_block_channel = pool.validated_pool().import_notification_stream() .map(|_| { EngineCommand::SealNewBlock { create_empty: false, @@ -260,7 +260,7 @@ mod tests { // this test checks that blocks are created as soon as transactions are imported into the pool. let (sender, receiver) = futures::channel::oneshot::channel(); let mut sender = Arc::new(Some(sender)); - let stream = pool.pool().import_notification_stream() + let stream = pool.pool().validated_pool().import_notification_stream() .map(move |_| { // we're only going to submit one tx so this fn will only be called once. let mut_sender = Arc::get_mut(&mut sender).unwrap(); diff --git a/client/consensus/manual-seal/src/seal_new_block.rs b/client/consensus/manual-seal/src/seal_new_block.rs index 53dc82d353..2b8d867ce3 100644 --- a/client/consensus/manual-seal/src/seal_new_block.rs +++ b/client/consensus/manual-seal/src/seal_new_block.rs @@ -92,7 +92,7 @@ pub async fn seal_new_block( SC: SelectChain, { let future = async { - if pool.status().ready == 0 && !create_empty { + if pool.validated_pool().status().ready == 0 && !create_empty { return Err(Error::EmptyTransactionPool) } diff --git a/client/consensus/slots/Cargo.toml b/client/consensus/slots/Cargo.toml index d500a0fdc7..67de0a54ed 100644 --- a/client/consensus/slots/Cargo.toml +++ b/client/consensus/slots/Cargo.toml @@ -19,7 +19,7 @@ sc-telemetry = { version = "2.0.0", path = "../../telemetry" } sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } sp-inherents = { version = "2.0.0", path = "../../../primitives/inherents" } futures = "0.3.1" -futures-timer = "2.0" +futures-timer = "3.0.1" parking_lot = "0.10.0" log = "0.4.8" diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index d35a554d3f..ef418d43ef 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -8,6 +8,7 @@ license = "GPL-3.0" [dependencies] parking_lot = "0.10.0" log = "0.4.8" +rand = "0.7" kvdb = "0.4.0" kvdb-rocksdb = { version = "0.5", optional = true } kvdb-memorydb = "0.4.0" diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs new file mode 100644 index 0000000000..9858a5c148 --- /dev/null +++ b/client/db/src/bench.rs @@ -0,0 +1,286 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! State backend that's useful for benchmarking + +use std::sync::Arc; +use std::path::PathBuf; +use std::cell::{Cell, RefCell}; +use rand::Rng; + +use hash_db::{Prefix, Hasher}; +use sp_trie::{MemoryDB, prefixed_key}; +use sp_core::storage::ChildInfo; +use sp_runtime::traits::{Block as BlockT, HasherFor}; +use sp_runtime::Storage; +use sp_state_machine::{DBValue, backend::Backend as StateBackend}; +use kvdb::{KeyValueDB, DBTransaction}; +use kvdb_rocksdb::{Database, DatabaseConfig}; + +type DbState = sp_state_machine::TrieBackend< + Arc>>, HasherFor +>; + +struct StorageDb { + db: Arc, + _block: std::marker::PhantomData, +} + +impl sp_state_machine::Storage> for StorageDb { + fn get(&self, key: &Block::Hash, prefix: Prefix) -> Result, String> { + let key = prefixed_key::>(key, prefix); + self.db.get(0, &key) + .map_err(|e| format!("Database backend error: {:?}", e)) + } +} + +/// State that manages the backend database reference. Allows runtime to control the database. +pub struct BenchmarkingState { + path: PathBuf, + root: Cell, + state: RefCell>>, + db: Cell>>, + genesis: as StateBackend>>::Transaction, +} + +impl BenchmarkingState { + /// Create a new instance that creates a database in a temporary dir. + pub fn new(genesis: Storage) -> Result { + let temp_dir = PathBuf::from(std::env::temp_dir()); + let name: String = rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(10).collect(); + let path = temp_dir.join(&name); + + let mut root = B::Hash::default(); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + + std::fs::create_dir(&path).map_err(|_| String::from("Error creating temp dir"))?; + let mut state = BenchmarkingState { + state: RefCell::new(None), + db: Cell::new(None), + path, + root: Cell::new(root), + genesis: Default::default(), + }; + + state.reopen()?; + let child_delta = genesis.children.into_iter().map(|(storage_key, child_content)| ( + storage_key, + child_content.data.into_iter().map(|(k, v)| (k, Some(v))), + child_content.child_info + )); + let (root, transaction) = state.state.borrow_mut().as_mut().unwrap().full_storage_root( + genesis.top.into_iter().map(|(k, v)| (k, Some(v))), + child_delta, + ); + state.genesis = transaction.clone(); + state.commit(root, transaction)?; + Ok(state) + } + + fn reopen(&self) -> Result<(), String> { + *self.state.borrow_mut() = None; + self.db.set(None); + let db_config = DatabaseConfig::with_columns(1); + let path = self.path.to_str() + .ok_or_else(|| String::from("Invalid database path"))?; + let db = Arc::new(Database::open(&db_config, &path).map_err(|e| format!("Error opening database: {:?}", e))?); + self.db.set(Some(db.clone())); + let storage_db = Arc::new(StorageDb:: { db, _block: Default::default() }); + *self.state.borrow_mut() = Some(DbState::::new(storage_db, self.root.get())); + Ok(()) + } + + fn kill(&self) -> Result<(), String> { + self.db.set(None); + *self.state.borrow_mut() = None; + let mut root = B::Hash::default(); + let mut mdb = MemoryDB::>::default(); + sp_state_machine::TrieDBMut::>::new(&mut mdb, &mut root); + self.root.set(root); + + std::fs::remove_dir_all(&self.path).map_err(|_| "Error removing database dir".into()) + } +} + +impl Drop for BenchmarkingState { + fn drop(&mut self) { + self.kill().ok(); + } +} + +fn state_err() -> String { + "State is not open".into() +} + +impl StateBackend> for BenchmarkingState { + type Error = as StateBackend>>::Error; + type Transaction = as StateBackend>>::Transaction; + type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; + + fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key) + } + + fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key) + } + + fn child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(storage_key, child_info, key) + } + + fn exists_storage(&self, key: &[u8]) -> Result { + self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) + } + + fn exists_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result { + self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(storage_key, child_info, key) + } + + fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key) + } + + fn next_child_storage_key( + &self, + storage_key: &[u8], + child_info: ChildInfo, + key: &[u8], + ) -> Result>, Self::Error> { + self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(storage_key, child_info, key) + } + + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + if let Some(ref state) = *self.state.borrow() { + state.for_keys_with_prefix(prefix, f) + } + } + + fn for_key_values_with_prefix(&self, prefix: &[u8], f: F) { + if let Some(ref state) = *self.state.borrow() { + state.for_key_values_with_prefix(prefix, f) + } + } + + fn for_keys_in_child_storage( + &self, + storage_key: &[u8], + child_info: ChildInfo, + f: F, + ) { + if let Some(ref state) = *self.state.borrow() { + state.for_keys_in_child_storage(storage_key, child_info, f) + } + } + + fn for_child_keys_with_prefix( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + f: F, + ) { + if let Some(ref state) = *self.state.borrow() { + state.for_child_keys_with_prefix(storage_key, child_info, prefix, f) + } + } + + fn storage_root(&self, delta: I) -> (B::Hash, Self::Transaction) where + I: IntoIterator, Option>)> + { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.storage_root(delta)) + } + + fn child_storage_root( + &self, + storage_key: &[u8], + child_info: ChildInfo, + delta: I, + ) -> (B::Hash, bool, Self::Transaction) where + I: IntoIterator, Option>)>, + { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_storage_root(storage_key, child_info, delta)) + } + + fn pairs(&self) -> Vec<(Vec, Vec)> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.pairs()) + } + + fn keys(&self, prefix: &[u8]) -> Vec> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.keys(prefix)) + } + + fn child_keys( + &self, + storage_key: &[u8], + child_info: ChildInfo, + prefix: &[u8], + ) -> Vec> { + self.state.borrow().as_ref().map_or(Default::default(), |s| s.child_keys(storage_key, child_info, prefix)) + } + + fn as_trie_backend(&mut self) + -> Option<&sp_state_machine::TrieBackend>> + { + None + } + + fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) + -> Result<(), Self::Error> + { + if let Some(db) = self.db.take() { + let mut db_transaction = DBTransaction::new(); + + for (key, (val, rc)) in transaction.drain() { + if rc > 0 { + db_transaction.put(0, &key, &val); + } else if rc < 0 { + db_transaction.delete(0, &key); + } + } + db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; + self.root.set(storage_root); + } else { + return Err("Trying to commit to a closed db".into()) + } + self.reopen() + } + + fn wipe(&self) -> Result<(), Self::Error> { + self.kill()?; + self.reopen()?; + self.commit(self.root.get(), self.genesis.clone())?; + Ok(()) + } +} + +impl std::fmt::Debug for BenchmarkingState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DB at {:?}", self.path) + } +} + diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index 36cf7dacf4..b641234281 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -29,6 +29,9 @@ pub mod light; pub mod offchain; +#[cfg(any(feature = "kvdb-rocksdb", test))] +pub mod bench; + mod children; mod cache; mod changes_tries_storage; @@ -80,6 +83,9 @@ use crate::stats::StateUsageStats; use log::{trace, debug, warn}; pub use sc_state_db::PruningMode; +#[cfg(any(feature = "kvdb-rocksdb", test))] +pub use bench::BenchmarkingState; + #[cfg(feature = "test-helpers")] use sc_client::in_mem::Backend as InMemoryBackend; @@ -1466,6 +1472,7 @@ impl sc_client_api::backend::Backend for Backend { average_transaction_size: io_stats.avg_transaction_size() as u64, state_reads: state_stats.reads.ops, state_reads_cache: state_stats.cache_reads.ops, + state_writes: state_stats.writes.ops, }, }) } diff --git a/client/db/src/light.rs b/client/db/src/light.rs index 8fb220973f..bea08d6467 100644 --- a/client/db/src/light.rs +++ b/client/db/src/light.rs @@ -592,6 +592,7 @@ impl LightBlockchainStorage for LightStorage // Light client does not track those state_reads: 0, state_reads_cache: 0, + state_writes: 0, } }) } diff --git a/client/db/src/utils.rs b/client/db/src/utils.rs index f7f51d7f6d..534cbb2197 100644 --- a/client/db/src/utils.rs +++ b/client/db/src/utils.rs @@ -320,7 +320,9 @@ pub fn require_header( id: BlockId, ) -> sp_blockchain::Result { read_header(db, col_index, col, id) - .and_then(|header| header.ok_or_else(|| sp_blockchain::Error::UnknownBlock(format!("{}", id)))) + .and_then(|header| header.ok_or_else(|| + sp_blockchain::Error::UnknownBlock(format!("Require header: {}", id)) + )) } /// Read meta from the database. diff --git a/client/executor/common/src/wasm_runtime.rs b/client/executor/common/src/wasm_runtime.rs index 0733350f4c..7af6c2bd53 100644 --- a/client/executor/common/src/wasm_runtime.rs +++ b/client/executor/common/src/wasm_runtime.rs @@ -17,21 +17,18 @@ //! Definitions for a wasm runtime. use crate::error::Error; -use sp_wasm_interface::Function; +use sp_wasm_interface::{Function, Value}; /// A trait that defines an abstract wasm runtime. /// /// This can be implemented by an execution engine. pub trait WasmRuntime { - /// Attempt to update the number of heap pages available during execution. - /// - /// Returns false if the update cannot be applied. The function is guaranteed to return true if - /// the heap pages would not change from its current value. - fn update_heap_pages(&mut self, heap_pages: u64) -> bool; - /// Return the host functions that are registered for this Wasm runtime. fn host_functions(&self) -> &[&'static dyn Function]; /// Call a method in the Substrate runtime by name. Returns the encoded result on success. fn call(&mut self, method: &str, data: &[u8]) -> Result, Error>; + + /// Get the value from a global with the given `name`. + fn get_global_val(&self, name: &str) -> Result, Error>; } diff --git a/client/executor/runtime-test/src/lib.rs b/client/executor/runtime-test/src/lib.rs index b183398c02..38a16ae39e 100644 --- a/client/executor/runtime-test/src/lib.rs +++ b/client/executor/runtime-test/src/lib.rs @@ -275,6 +275,20 @@ sp_core::wasm_export_functions! { data.to_vec() } + + // Check that the heap at `heap_base + offset` don't contains the test message. + // After the check succeeds the test message is written into the heap. + // + // It is expected that the given pointer is not allocated. + fn check_and_set_in_heap(heap_base: u32, offset: u32) { + let test_message = b"Hello invalid heap memory"; + let ptr = unsafe { (heap_base + offset) as *mut u8 }; + + let message_slice = unsafe { sp_std::slice::from_raw_parts_mut(ptr, test_message.len()) }; + + assert_ne!(test_message, message_slice); + message_slice.copy_from_slice(test_message); + } } #[cfg(not(feature = "std"))] diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index 3f3d9f69e1..c0516d3ac7 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -88,7 +88,7 @@ fn call_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 68\")" + "Other(\"Wasm execution trapped: call to a missing function env:missing_external\")" ), } } @@ -117,7 +117,7 @@ fn call_yet_another_not_existing_function(wasm_method: WasmExecutionMethod) { #[cfg(feature = "wasmtime")] WasmExecutionMethod::Compiled => assert_eq!( &format!("{:?}", e), - "Other(\"call to undefined external function with index 69\")" + "Other(\"Wasm execution trapped: call to a missing function env:yet_another_missing_external\")" ), } } @@ -565,3 +565,26 @@ fn restoration_of_globals(wasm_method: WasmExecutionMethod) { let res = instance.call("allocates_huge_stack_array", &false.encode()); assert!(res.is_ok()); } + +#[test_case(WasmExecutionMethod::Interpreted)] +fn heap_is_reset_between_calls(wasm_method: WasmExecutionMethod) { + let mut instance = crate::wasm_runtime::create_wasm_runtime_with_code( + wasm_method, + 1024, + &WASM_BINARY[..], + HostFunctions::host_functions(), + true, + ).expect("Creates instance"); + + let heap_base = instance.get_global_val("__heap_base") + .expect("`__heap_base` is valid") + .expect("`__heap_base` exists") + .as_i32() + .expect("`__heap_base` is an `i32`"); + + let params = (heap_base as u32, 512u32 * 64 * 1024).encode(); + instance.call("check_and_set_in_heap", ¶ms).unwrap(); + + // Cal it a second time to check that the heap was freed. + instance.call("check_and_set_in_heap", ¶ms).unwrap(); +} diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index b8966c3af2..9d54246ee0 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -42,6 +42,8 @@ pub enum WasmExecutionMethod { /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { runtime: Box, + /// The number of WebAssembly heap pages this instance was created with. + heap_pages: u64, /// Runtime version according to `Core_version`. version: RuntimeVersion, } @@ -122,7 +124,7 @@ impl RuntimesCache { Entry::Occupied(o) => { let result = o.into_mut(); if let Ok(ref mut cached_runtime) = result { - let heap_pages_changed = !cached_runtime.runtime.update_heap_pages(heap_pages); + let heap_pages_changed = cached_runtime.heap_pages != heap_pages; let host_functions_changed = cached_runtime.runtime.host_functions() != host_functions; if heap_pages_changed || host_functions_changed { @@ -236,6 +238,7 @@ fn create_versioned_wasm_runtime( Ok(VersionedRuntime { runtime, version, + heap_pages, }) } diff --git a/client/executor/wasmi/src/lib.rs b/client/executor/wasmi/src/lib.rs index 6fbfdbc1cc..b90c0f05f5 100644 --- a/client/executor/wasmi/src/lib.rs +++ b/client/executor/wasmi/src/lib.rs @@ -73,8 +73,7 @@ impl<'a> sandbox::SandboxCapabilities for FunctionExecutor<'a> { invoke_args_len: WordSize, state: u32, func_idx: sandbox::SupervisorFuncIndex, - ) -> Result - { + ) -> Result { let result = wasmi::FuncInstance::invoke( dispatch_thunk, &[ @@ -536,7 +535,6 @@ struct StateSnapshot { data_segments: Vec<(u32, Vec)>, /// The list of all global mutable variables of the module in their sequential order. global_mut_values: Vec, - heap_pages: u64, } impl StateSnapshot { @@ -544,7 +542,6 @@ impl StateSnapshot { fn take( module_instance: &ModuleRef, data_segments: Vec, - heap_pages: u64, ) -> Option { let prepared_segments = data_segments .into_iter() @@ -590,7 +587,6 @@ impl StateSnapshot { Some(Self { data_segments: prepared_segments, global_mut_values, - heap_pages, }) } @@ -646,10 +642,6 @@ pub struct WasmiRuntime { } impl WasmRuntime for WasmiRuntime { - fn update_heap_pages(&mut self, heap_pages: u64) -> bool { - self.state_snapshot.heap_pages == heap_pages - } - fn host_functions(&self) -> &[&'static dyn Function] { &self.host_functions } @@ -677,6 +669,19 @@ impl WasmRuntime for WasmiRuntime { &self.missing_functions, ) } + + fn get_global_val(&self, name: &str) -> Result, Error> { + match self.instance.export_by_name(name) { + Some(global) => Ok(Some( + global + .as_global() + .ok_or_else(|| format!("`{}` is not a global", name))? + .get() + .into() + )), + None => Ok(None), + } + } } pub fn create_instance( @@ -702,7 +707,7 @@ pub fn create_instance( ).map_err(|e| WasmError::Instantiation(e.to_string()))?; // Take state snapshot before executing anything. - let state_snapshot = StateSnapshot::take(&instance, data_segments, heap_pages) + let state_snapshot = StateSnapshot::take(&instance, data_segments) .expect( "`take` returns `Err` if the module is not valid; we already loaded module above, thus the `Module` is proven to be valid at this point; diff --git a/client/executor/wasmtime/Cargo.toml b/client/executor/wasmtime/Cargo.toml index 30d3a5dc87..eb41adb271 100644 --- a/client/executor/wasmtime/Cargo.toml +++ b/client/executor/wasmtime/Cargo.toml @@ -16,14 +16,7 @@ sp-runtime-interface = { version = "2.0.0", path = "../../../primitives/runtime- sp-core = { version = "2.0.0", path = "../../../primitives/core" } sp-allocator = { version = "2.0.0", path = "../../../primitives/allocator" } -cranelift-codegen = "0.50" -cranelift-entity = "0.50" -cranelift-frontend = "0.50" -cranelift-native = "0.50" -cranelift-wasm = "0.50" -wasmtime-environ = "0.8" -wasmtime-jit = "0.8" -wasmtime-runtime = "0.8" +wasmtime = "0.11" [dev-dependencies] assert_matches = "1.3.0" diff --git a/client/executor/wasmtime/src/function_executor.rs b/client/executor/wasmtime/src/function_executor.rs deleted file mode 100644 index b4971f8b8a..0000000000 --- a/client/executor/wasmtime/src/function_executor.rs +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -use sp_allocator::FreeingBumpHeapAllocator; -use sc_executor_common::error::{Error, Result}; -use sc_executor_common::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex}; -use crate::util::{ - checked_range, cranelift_ir_signature, read_memory_into, write_memory_from, -}; - -use codec::{Decode, Encode}; -use cranelift_codegen::ir; -use cranelift_codegen::isa::TargetFrontendConfig; -use log::trace; -use sp_core::sandbox as sandbox_primitives; -use std::{cmp, mem, ptr}; -use wasmtime_environ::translate_signature; -use wasmtime_jit::{ActionError, Compiler}; -use wasmtime_runtime::{Export, VMCallerCheckedAnyfunc, VMContext, wasmtime_call_trampoline}; -use sp_wasm_interface::{ - FunctionContext, MemoryId, Pointer, Result as WResult, Sandbox, Signature, Value, ValueType, - WordSize, -}; - -/// Wrapper type for pointer to a Wasm table entry. -/// -/// The wrapper type is used to ensure that the function reference is valid as it must be unsafely -/// dereferenced from within the safe method `::invoke`. -#[derive(Clone, Copy)] -pub struct SupervisorFuncRef(*const VMCallerCheckedAnyfunc); - -/// The state required to construct a FunctionExecutor context. The context only lasts for one host -/// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make -/// many different host calls that must share state. -/// -/// This is stored as part of the host state of the "env" Wasmtime instance. -pub struct FunctionExecutorState { - sandbox_store: sandbox::Store, - heap: FreeingBumpHeapAllocator, -} - -impl FunctionExecutorState { - /// Constructs a new `FunctionExecutorState`. - pub fn new(heap_base: u32) -> Self { - FunctionExecutorState { - sandbox_store: sandbox::Store::new(), - heap: FreeingBumpHeapAllocator::new(heap_base), - } - } - - /// Returns a mutable reference to the heap allocator. - pub fn heap(&mut self) -> &mut FreeingBumpHeapAllocator { - &mut self.heap - } -} - -/// A `FunctionExecutor` implements `FunctionContext` for making host calls from a Wasmtime -/// runtime. The `FunctionExecutor` exists only for the lifetime of the call and borrows state from -/// a longer-living `FunctionExecutorState`. -pub struct FunctionExecutor<'a> { - compiler: &'a mut Compiler, - sandbox_store: &'a mut sandbox::Store, - heap: &'a mut FreeingBumpHeapAllocator, - memory: &'a mut [u8], - table: Option<&'a [VMCallerCheckedAnyfunc]>, -} - -impl<'a> FunctionExecutor<'a> { - /// Construct a new `FunctionExecutor`. - /// - /// The vmctx MUST come from a call to a function in the "env" module. - /// The state MUST be looked up from the host state of the "env" module. - pub unsafe fn new( - vmctx: *mut VMContext, - compiler: &'a mut Compiler, - state: &'a mut FunctionExecutorState, - ) -> Result - { - let memory = match (*vmctx).lookup_global_export("memory") { - Some(Export::Memory { definition, vmctx: _, memory: _ }) => - std::slice::from_raw_parts_mut( - (*definition).base, - (*definition).current_length, - ), - _ => return Err(Error::InvalidMemoryReference), - }; - let table = match (*vmctx).lookup_global_export("__indirect_function_table") { - Some(Export::Table { definition, vmctx: _, table: _ }) => - Some(std::slice::from_raw_parts( - (*definition).base as *const VMCallerCheckedAnyfunc, - (*definition).current_elements as usize, - )), - _ => None, - }; - Ok(FunctionExecutor { - compiler, - sandbox_store: &mut state.sandbox_store, - heap: &mut state.heap, - memory, - table, - }) - } -} - -impl<'a> SandboxCapabilities for FunctionExecutor<'a> { - type SupervisorFuncRef = SupervisorFuncRef; - - fn invoke( - &mut self, - dispatch_thunk: &Self::SupervisorFuncRef, - invoke_args_ptr: Pointer, - invoke_args_len: WordSize, - state: u32, - func_idx: SupervisorFuncIndex, - ) -> Result - { - let func_ptr = unsafe { (*dispatch_thunk.0).func_ptr }; - let vmctx = unsafe { (*dispatch_thunk.0).vmctx }; - - // The following code is based on the wasmtime_jit::Context::invoke. - let value_size = mem::size_of::(); - let (signature, mut values_vec) = generate_signature_and_args( - &[ - Value::I32(u32::from(invoke_args_ptr) as i32), - Value::I32(invoke_args_len as i32), - Value::I32(state as i32), - Value::I32(usize::from(func_idx) as i32), - ], - Some(ValueType::I64), - self.compiler.frontend_config(), - ); - - // Get the trampoline to call for this function. - let exec_code_buf = self.compiler - .get_published_trampoline(func_ptr, &signature, value_size) - .map_err(ActionError::Setup) - .map_err(|e| Error::Other(e.to_string()))?; - - // Call the trampoline. - if let Err(message) = unsafe { - wasmtime_call_trampoline( - vmctx, - exec_code_buf, - values_vec.as_mut_ptr() as *mut u8, - ) - } { - return Err(Error::Other(message)); - } - - // Load the return value out of `values_vec`. - Ok(unsafe { ptr::read(values_vec.as_ptr() as *const i64) }) - } -} - -impl<'a> FunctionContext for FunctionExecutor<'a> { - fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> WResult<()> { - read_memory_into(self.memory, address, dest).map_err(|e| e.to_string()) - } - - fn write_memory(&mut self, address: Pointer, data: &[u8]) -> WResult<()> { - write_memory_from(self.memory, address, data).map_err(|e| e.to_string()) - } - - fn allocate_memory(&mut self, size: WordSize) -> WResult> { - self.heap.allocate(self.memory, size).map_err(|e| e.to_string()) - } - - fn deallocate_memory(&mut self, ptr: Pointer) -> WResult<()> { - self.heap.deallocate(self.memory, ptr).map_err(|e| e.to_string()) - } - - fn sandbox(&mut self) -> &mut dyn Sandbox { - self - } -} - -impl<'a> Sandbox for FunctionExecutor<'a> { - fn memory_get( - &mut self, - memory_id: MemoryId, - offset: WordSize, - buf_ptr: Pointer, - buf_len: WordSize, - ) -> WResult - { - let sandboxed_memory = self.sandbox_store.memory(memory_id) - .map_err(|e| e.to_string())?; - sandboxed_memory.with_direct_access(|memory| { - let len = buf_len as usize; - let src_range = match checked_range(offset as usize, len, memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - let dst_range = match checked_range(buf_ptr.into(), len, self.memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - &mut self.memory[dst_range].copy_from_slice(&memory[src_range]); - Ok(sandbox_primitives::ERR_OK) - }) - } - - fn memory_set( - &mut self, - memory_id: MemoryId, - offset: WordSize, - val_ptr: Pointer, - val_len: WordSize, - ) -> WResult - { - let sandboxed_memory = self.sandbox_store.memory(memory_id) - .map_err(|e| e.to_string())?; - sandboxed_memory.with_direct_access_mut(|memory| { - let len = val_len as usize; - let src_range = match checked_range(val_ptr.into(), len, self.memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - let dst_range = match checked_range(offset as usize, len, memory.len()) { - Some(range) => range, - None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), - }; - &mut memory[dst_range].copy_from_slice(&self.memory[src_range]); - Ok(sandbox_primitives::ERR_OK) - }) - } - - fn memory_teardown(&mut self, memory_id: MemoryId) - -> WResult<()> - { - self.sandbox_store.memory_teardown(memory_id).map_err(|e| e.to_string()) - } - - fn memory_new(&mut self, initial: u32, maximum: MemoryId) -> WResult { - self.sandbox_store.new_memory(initial, maximum).map_err(|e| e.to_string()) - } - - fn invoke( - &mut self, - instance_id: u32, - export_name: &str, - args: &[u8], - return_val: Pointer, - return_val_len: u32, - state: u32, - ) -> WResult { - trace!(target: "sp-sandbox", "invoke, instance_idx={}", instance_id); - - // Deserialize arguments and convert them into wasmi types. - let args = Vec::::decode(&mut &args[..]) - .map_err(|_| "Can't decode serialized arguments for the invocation")? - .into_iter() - .map(Into::into) - .collect::>(); - - let instance = self.sandbox_store.instance(instance_id).map_err(|e| e.to_string())?; - let result = instance.invoke(export_name, &args, self, state); - - match result { - Ok(None) => Ok(sandbox_primitives::ERR_OK), - Ok(Some(val)) => { - // Serialize return value and write it back into the memory. - sp_wasm_interface::ReturnValue::Value(val.into()).using_encoded(|val| { - if val.len() > return_val_len as usize { - Err("Return value buffer is too small")?; - } - FunctionContext::write_memory(self, return_val, val)?; - Ok(sandbox_primitives::ERR_OK) - }) - } - Err(_) => Ok(sandbox_primitives::ERR_EXECUTION), - } - } - - fn instance_teardown(&mut self, instance_id: u32) -> WResult<()> { - self.sandbox_store.instance_teardown(instance_id).map_err(|e| e.to_string()) - } - - fn instance_new(&mut self, dispatch_thunk_id: u32, wasm: &[u8], raw_env_def: &[u8], state: u32) - -> WResult - { - // Extract a dispatch thunk from instance's table by the specified index. - let dispatch_thunk = { - let table = self.table.as_ref() - .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")?; - let func_ref = table.get(dispatch_thunk_id as usize) - .ok_or_else(|| "dispatch_thunk_idx is out of the table bounds")?; - SupervisorFuncRef(func_ref) - }; - - let guest_env = match sandbox::GuestEnvironment::decode(&self.sandbox_store, raw_env_def) { - Ok(guest_env) => guest_env, - Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), - }; - - let instance_idx_or_err_code = - match sandbox::instantiate(self, dispatch_thunk, wasm, guest_env, state) - .map(|i| i.register(&mut self.sandbox_store)) - { - Ok(instance_idx) => instance_idx, - Err(sandbox::InstantiationError::StartTrapped) => - sandbox_primitives::ERR_EXECUTION, - Err(_) => sandbox_primitives::ERR_MODULE, - }; - - Ok(instance_idx_or_err_code as u32) - } - - fn get_global_val( - &self, - instance_idx: u32, - name: &str, - ) -> WResult> { - self.sandbox_store - .instance(instance_idx) - .map(|i| i.get_global_val(name)) - .map_err(|e| e.to_string()) - } -} - -// The storage for a Wasmtime invocation argument. -#[derive(Debug, Default, Copy, Clone)] -#[repr(C, align(8))] -struct VMInvokeArgument([u8; 8]); - -fn generate_signature_and_args( - args: &[Value], - result_type: Option, - frontend_config: TargetFrontendConfig, -) -> (ir::Signature, Vec) -{ - // This code is based on the wasmtime_jit::Context::invoke. - - let param_types = args.iter() - .map(|arg| arg.value_type()) - .collect::>(); - let signature = translate_signature( - cranelift_ir_signature( - Signature::new(param_types, result_type), - &frontend_config.default_call_conv - ), - frontend_config.pointer_type() - ); - - let mut values_vec = vec![ - VMInvokeArgument::default(); - cmp::max(args.len(), result_type.iter().len()) - ]; - - // Store the argument values into `values_vec`. - for (index, arg) in args.iter().enumerate() { - unsafe { - let ptr = values_vec.as_mut_ptr().add(index); - - match arg { - Value::I32(x) => ptr::write(ptr as *mut i32, *x), - Value::I64(x) => ptr::write(ptr as *mut i64, *x), - Value::F32(x) => ptr::write(ptr as *mut u32, *x), - Value::F64(x) => ptr::write(ptr as *mut u64, *x), - } - } - } - - (signature, values_vec) -} - diff --git a/client/executor/wasmtime/src/host.rs b/client/executor/wasmtime/src/host.rs new file mode 100644 index 0000000000..e0cc6ecc9a --- /dev/null +++ b/client/executor/wasmtime/src/host.rs @@ -0,0 +1,349 @@ +// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! This module defines `HostState` and `HostContext` structs which provide logic and state +//! required for execution of host. + +use crate::instance_wrapper::InstanceWrapper; +use crate::util; +use std::cell::RefCell; +use log::trace; +use codec::{Encode, Decode}; +use sp_allocator::FreeingBumpHeapAllocator; +use sc_executor_common::error::Result; +use sc_executor_common::sandbox::{self, SandboxCapabilities, SupervisorFuncIndex}; +use sp_core::sandbox as sandbox_primitives; +use sp_wasm_interface::{FunctionContext, MemoryId, Pointer, Sandbox, WordSize}; +use wasmtime::{Func, Val}; + +/// Wrapper type for pointer to a Wasm table entry. +/// +/// The wrapper type is used to ensure that the function reference is valid as it must be unsafely +/// dereferenced from within the safe method `::invoke`. +#[derive(Clone)] +pub struct SupervisorFuncRef(Func); + +/// The state required to construct a HostContext context. The context only lasts for one host +/// call, whereas the state is maintained for the duration of a Wasm runtime call, which may make +/// many different host calls that must share state. +pub struct HostState { + // We need some interior mutability here since the host state is shared between all host + // function handlers and the wasmtime backend's `impl WasmRuntime`. + // + // Furthermore, because of recursive calls (e.g. runtime can create and call an sandboxed + // instance which in turn can call the runtime back) we have to be very careful with borrowing + // those. + // + // Basically, most of the interactions should do temporary borrow immediately releasing the + // borrow after performing necessary queries/changes. + sandbox_store: RefCell>, + allocator: RefCell, + instance: InstanceWrapper, +} + +impl HostState { + /// Constructs a new `HostState`. + pub fn new(allocator: FreeingBumpHeapAllocator, instance: InstanceWrapper) -> Self { + HostState { + sandbox_store: RefCell::new(sandbox::Store::new()), + allocator: RefCell::new(allocator), + instance, + } + } + + /// Destruct the host state and extract the `InstanceWrapper` passed at the creation. + pub fn into_instance(self) -> InstanceWrapper { + self.instance + } + + /// Materialize `HostContext` that can be used to invoke a substrate host `dyn Function`. + pub fn materialize<'a>(&'a self) -> HostContext<'a> { + HostContext(self) + } +} + +/// A `HostContext` implements `FunctionContext` for making host calls from a Wasmtime +/// runtime. The `HostContext` exists only for the lifetime of the call and borrows state from +/// a longer-living `HostState`. +pub struct HostContext<'a>(&'a HostState); + +impl<'a> std::ops::Deref for HostContext<'a> { + type Target = HostState; + fn deref(&self) -> &HostState { + self.0 + } +} + +impl<'a> SandboxCapabilities for HostContext<'a> { + type SupervisorFuncRef = SupervisorFuncRef; + + fn invoke( + &mut self, + dispatch_thunk: &Self::SupervisorFuncRef, + invoke_args_ptr: Pointer, + invoke_args_len: WordSize, + state: u32, + func_idx: SupervisorFuncIndex, + ) -> Result { + let result = dispatch_thunk.0.call(&[ + Val::I32(u32::from(invoke_args_ptr) as i32), + Val::I32(invoke_args_len as i32), + Val::I32(state as i32), + Val::I32(usize::from(func_idx) as i32), + ]); + match result { + Ok(ret_vals) => { + let ret_val = if ret_vals.len() != 1 { + return Err(format!( + "Supervisor function returned {} results, expected 1", + ret_vals.len() + ) + .into()); + } else { + &ret_vals[0] + }; + + if let Some(ret_val) = ret_val.i64() { + Ok(ret_val) + } else { + return Err("Supervisor function returned unexpected result!".into()); + } + } + Err(err) => Err(err.message().to_string().into()), + } + } +} + +impl<'a> sp_wasm_interface::FunctionContext for HostContext<'a> { + fn read_memory_into( + &self, + address: Pointer, + dest: &mut [u8], + ) -> sp_wasm_interface::Result<()> { + self.instance + .read_memory_into(address, dest) + .map_err(|e| e.to_string()) + } + + fn write_memory(&mut self, address: Pointer, data: &[u8]) -> sp_wasm_interface::Result<()> { + self.instance + .write_memory_from(address, data) + .map_err(|e| e.to_string()) + } + + fn allocate_memory(&mut self, size: WordSize) -> sp_wasm_interface::Result> { + self.instance + .allocate(&mut *self.allocator.borrow_mut(), size) + .map_err(|e| e.to_string()) + } + + fn deallocate_memory(&mut self, ptr: Pointer) -> sp_wasm_interface::Result<()> { + self.instance + .deallocate(&mut *self.allocator.borrow_mut(), ptr) + .map_err(|e| e.to_string()) + } + + fn sandbox(&mut self) -> &mut dyn Sandbox { + self + } +} + +impl<'a> Sandbox for HostContext<'a> { + fn memory_get( + &mut self, + memory_id: MemoryId, + offset: WordSize, + buf_ptr: Pointer, + buf_len: WordSize, + ) -> sp_wasm_interface::Result { + let sandboxed_memory = self + .sandbox_store + .borrow() + .memory(memory_id) + .map_err(|e| e.to_string())?; + sandboxed_memory.with_direct_access(|sandboxed_memory| { + let len = buf_len as usize; + let src_range = match util::checked_range(offset as usize, len, sandboxed_memory.len()) + { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + let supervisor_mem_size = self.instance.memory_size() as usize; + let dst_range = match util::checked_range(buf_ptr.into(), len, supervisor_mem_size) { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + self.instance + .write_memory_from( + Pointer::new(dst_range.start as u32), + &sandboxed_memory[src_range], + ) + .expect("ranges are checked above; write can't fail; qed"); + Ok(sandbox_primitives::ERR_OK) + }) + } + + fn memory_set( + &mut self, + memory_id: MemoryId, + offset: WordSize, + val_ptr: Pointer, + val_len: WordSize, + ) -> sp_wasm_interface::Result { + let sandboxed_memory = self + .sandbox_store + .borrow() + .memory(memory_id) + .map_err(|e| e.to_string())?; + sandboxed_memory.with_direct_access_mut(|sandboxed_memory| { + let len = val_len as usize; + let supervisor_mem_size = self.instance.memory_size() as usize; + let src_range = match util::checked_range(val_ptr.into(), len, supervisor_mem_size) { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + let dst_range = match util::checked_range(offset as usize, len, sandboxed_memory.len()) + { + Some(range) => range, + None => return Ok(sandbox_primitives::ERR_OUT_OF_BOUNDS), + }; + self.instance + .read_memory_into( + Pointer::new(src_range.start as u32), + &mut sandboxed_memory[dst_range], + ) + .expect("ranges are checked above; read can't fail; qed"); + Ok(sandbox_primitives::ERR_OK) + }) + } + + fn memory_teardown(&mut self, memory_id: MemoryId) -> sp_wasm_interface::Result<()> { + self.sandbox_store + .borrow_mut() + .memory_teardown(memory_id) + .map_err(|e| e.to_string()) + } + + fn memory_new(&mut self, initial: u32, maximum: MemoryId) -> sp_wasm_interface::Result { + self.sandbox_store + .borrow_mut() + .new_memory(initial, maximum) + .map_err(|e| e.to_string()) + } + + fn invoke( + &mut self, + instance_id: u32, + export_name: &str, + args: &[u8], + return_val: Pointer, + return_val_len: u32, + state: u32, + ) -> sp_wasm_interface::Result { + trace!(target: "sp-sandbox", "invoke, instance_idx={}", instance_id); + + // Deserialize arguments and convert them into wasmi types. + let args = Vec::::decode(&mut &args[..]) + .map_err(|_| "Can't decode serialized arguments for the invocation")? + .into_iter() + .map(Into::into) + .collect::>(); + + let instance = self + .sandbox_store + .borrow() + .instance(instance_id) + .map_err(|e| e.to_string())?; + let result = instance.invoke(export_name, &args, self, state); + + match result { + Ok(None) => Ok(sandbox_primitives::ERR_OK), + Ok(Some(val)) => { + // Serialize return value and write it back into the memory. + sp_wasm_interface::ReturnValue::Value(val.into()).using_encoded(|val| { + if val.len() > return_val_len as usize { + Err("Return value buffer is too small")?; + } + ::write_memory(self, return_val, val) + .map_err(|_| "can't write return value")?; + Ok(sandbox_primitives::ERR_OK) + }) + } + Err(_) => Ok(sandbox_primitives::ERR_EXECUTION), + } + } + + fn instance_teardown(&mut self, instance_id: u32) -> sp_wasm_interface::Result<()> { + self.sandbox_store + .borrow_mut() + .instance_teardown(instance_id) + .map_err(|e| e.to_string()) + } + + fn instance_new( + &mut self, + dispatch_thunk_id: u32, + wasm: &[u8], + raw_env_def: &[u8], + state: u32, + ) -> sp_wasm_interface::Result { + // Extract a dispatch thunk from the instance's table by the specified index. + let dispatch_thunk = { + let table_item = self + .instance + .table() + .as_ref() + .ok_or_else(|| "Runtime doesn't have a table; sandbox is unavailable")? + .get(dispatch_thunk_id); + + let func_ref = table_item + .ok_or_else(|| "dispatch_thunk_id is out of bounds")? + .funcref() + .ok_or_else(|| "dispatch_thunk_idx should be a funcref")? + .clone(); + SupervisorFuncRef(func_ref) + }; + + let guest_env = + match sandbox::GuestEnvironment::decode(&*self.sandbox_store.borrow(), raw_env_def) { + Ok(guest_env) => guest_env, + Err(_) => return Ok(sandbox_primitives::ERR_MODULE as u32), + }; + + let instance_idx_or_err_code = + match sandbox::instantiate(self, dispatch_thunk, wasm, guest_env, state) + .map(|i| i.register(&mut *self.sandbox_store.borrow_mut())) + { + Ok(instance_idx) => instance_idx, + Err(sandbox::InstantiationError::StartTrapped) => sandbox_primitives::ERR_EXECUTION, + Err(_) => sandbox_primitives::ERR_MODULE, + }; + + Ok(instance_idx_or_err_code as u32) + } + + fn get_global_val( + &self, + instance_idx: u32, + name: &str, + ) -> sp_wasm_interface::Result> { + self.sandbox_store + .borrow() + .instance(instance_idx) + .map(|i| i.get_global_val(name)) + .map_err(|e| e.to_string()) + } +} diff --git a/client/executor/wasmtime/src/imports.rs b/client/executor/wasmtime/src/imports.rs new file mode 100644 index 0000000000..349f84a0d7 --- /dev/null +++ b/client/executor/wasmtime/src/imports.rs @@ -0,0 +1,333 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::state_holder::StateHolder; +use sc_executor_common::error::WasmError; +use sp_wasm_interface::{Function, Value, ValueType}; +use std::any::Any; +use std::rc::Rc; +use wasmtime::{ + Callable, Extern, ExternType, Func, FuncType, ImportType, Limits, Memory, MemoryType, Module, + Trap, Val, +}; + +pub struct Imports { + /// Contains the index into `externs` where the memory import is stored if any. `None` if there + /// is none. + pub memory_import_index: Option, + pub externs: Vec, +} + +/// Goes over all imports of a module and prepares a vector of `Extern`s that can be used for +/// instantiation of the module. Returns an error if there are imports that cannot be satisfied. +pub fn resolve_imports( + state_holder: &StateHolder, + module: &Module, + host_functions: &[&'static dyn Function], + heap_pages: u32, + allow_missing_func_imports: bool, +) -> Result { + let mut externs = vec![]; + let mut memory_import_index = None; + for import_ty in module.imports() { + if import_ty.module() != "env" { + return Err(WasmError::Other(format!( + "host doesn't provide any imports from non-env module: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + + let resolved = match import_ty.name() { + "memory" => { + memory_import_index = Some(externs.len()); + resolve_memory_import(module, import_ty, heap_pages)? + } + _ => resolve_func_import( + module, + state_holder, + import_ty, + host_functions, + allow_missing_func_imports, + )?, + }; + externs.push(resolved); + } + Ok(Imports { + memory_import_index, + externs, + }) +} + +fn resolve_memory_import( + module: &Module, + import_ty: &ImportType, + heap_pages: u32, +) -> Result { + let requested_memory_ty = match import_ty.ty() { + ExternType::Memory(memory_ty) => memory_ty, + _ => { + return Err(WasmError::Other(format!( + "this import must be of memory type: {}:{}", + import_ty.module(), + import_ty.name() + ))) + } + }; + + // Increment the min (a.k.a initial) number of pages by `heap_pages` and check if it exceeds the + // maximum specified by the import. + let initial = requested_memory_ty + .limits() + .min() + .saturating_add(heap_pages); + if let Some(max) = requested_memory_ty.limits().max() { + if initial > max { + return Err(WasmError::Other(format!( + "incremented number of pages by heap_pages (total={}) is more than maximum requested\ + by the runtime wasm module {}", + initial, + max, + ))); + } + } + + let memory_ty = MemoryType::new(Limits::new(initial, requested_memory_ty.limits().max())); + let memory = Memory::new(module.store(), memory_ty); + Ok(Extern::Memory(memory)) +} + +fn resolve_func_import( + module: &Module, + state_holder: &StateHolder, + import_ty: &ImportType, + host_functions: &[&'static dyn Function], + allow_missing_func_imports: bool, +) -> Result { + let func_ty = match import_ty.ty() { + ExternType::Func(func_ty) => func_ty, + _ => { + return Err(WasmError::Other(format!( + "host doesn't provide any non function imports besides 'memory': {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + }; + + let host_func = match host_functions + .iter() + .find(|host_func| host_func.name() == import_ty.name()) + { + Some(host_func) => host_func, + None if allow_missing_func_imports => { + return Ok(MissingHostFuncHandler::new(import_ty).into_extern(module, func_ty)); + } + None => { + return Err(WasmError::Other(format!( + "host doesn't provide such function: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + }; + if !signature_matches(&func_ty, &wasmtime_func_sig(*host_func)) { + return Err(WasmError::Other(format!( + "signature mismatch for: {}:{}", + import_ty.module(), + import_ty.name() + ))); + } + + Ok(HostFuncHandler::new(&state_holder, *host_func).into_extern(module)) +} + +/// Returns `true` if `lhs` and `rhs` represent the same signature. +fn signature_matches(lhs: &wasmtime::FuncType, rhs: &wasmtime::FuncType) -> bool { + lhs.params() == rhs.params() && lhs.results() == rhs.results() +} + +/// This structure implements `Callable` and acts as a bridge between wasmtime and +/// substrate host functions. +struct HostFuncHandler { + state_holder: StateHolder, + host_func: &'static dyn Function, +} + +impl HostFuncHandler { + fn new(state_holder: &StateHolder, host_func: &'static dyn Function) -> Self { + Self { + state_holder: state_holder.clone(), + host_func, + } + } + + fn into_extern(self, module: &Module) -> Extern { + let func_ty = wasmtime_func_sig(self.host_func); + let func = Func::new(module.store(), func_ty, Rc::new(self)); + Extern::Func(func) + } +} + +impl Callable for HostFuncHandler { + fn call( + &self, + wasmtime_params: &[Val], + wasmtime_results: &mut [Val], + ) -> Result<(), wasmtime::Trap> { + let unwind_result = self.state_holder.with_context(|host_ctx| { + let mut host_ctx = host_ctx.expect( + "host functions can be called only from wasm instance; + wasm instance is always called initializing context; + therefore host_ctx cannot be None; + qed + ", + ); + // `into_value` panics if it encounters a value that doesn't fit into the values + // available in substrate. + // + // This, however, cannot happen since the signature of this function is created from + // a `dyn Function` signature of which cannot have a non substrate value by definition. + let mut params = wasmtime_params.iter().cloned().map(into_value); + + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + self.host_func.execute(&mut host_ctx, &mut params) + })) + }); + + let execution_result = match unwind_result { + Ok(execution_result) => execution_result, + Err(err) => return Err(Trap::new(stringify_panic_payload(err))), + }; + + match execution_result { + Ok(Some(ret_val)) => { + debug_assert!( + wasmtime_results.len() == 1, + "wasmtime function signature, therefore the number of results, should always \ + correspond to the number of results returned by the host function", + ); + wasmtime_results[0] = into_wasmtime_val(ret_val); + Ok(()) + } + Ok(None) => { + debug_assert!( + wasmtime_results.len() == 0, + "wasmtime function signature, therefore the number of results, should always \ + correspond to the number of results returned by the host function", + ); + Ok(()) + } + Err(msg) => Err(Trap::new(msg)), + } + } +} + +/// A `Callable` handler for missing functions. +struct MissingHostFuncHandler { + module: String, + name: String, +} + +impl MissingHostFuncHandler { + fn new(import_ty: &ImportType) -> Self { + Self { + module: import_ty.module().to_string(), + name: import_ty.name().to_string(), + } + } + + fn into_extern(self, module: &Module, func_ty: &FuncType) -> Extern { + let func = Func::new(module.store(), func_ty.clone(), Rc::new(self)); + Extern::Func(func) + } +} + +impl Callable for MissingHostFuncHandler { + fn call( + &self, + _wasmtime_params: &[Val], + _wasmtime_results: &mut [Val], + ) -> Result<(), wasmtime::Trap> { + Err(Trap::new(format!( + "call to a missing function {}:{}", + self.module, self.name + ))) + } +} + +fn wasmtime_func_sig(func: &dyn Function) -> wasmtime::FuncType { + let params = func + .signature() + .args + .iter() + .cloned() + .map(into_wasmtime_val_type) + .collect::>() + .into_boxed_slice(); + let results = func + .signature() + .return_value + .iter() + .cloned() + .map(into_wasmtime_val_type) + .collect::>() + .into_boxed_slice(); + wasmtime::FuncType::new(params, results) +} + +fn into_wasmtime_val_type(val_ty: ValueType) -> wasmtime::ValType { + match val_ty { + ValueType::I32 => wasmtime::ValType::I32, + ValueType::I64 => wasmtime::ValType::I64, + ValueType::F32 => wasmtime::ValType::F32, + ValueType::F64 => wasmtime::ValType::F64, + } +} + +/// Converts a `Val` into a substrate runtime interface `Value`. +/// +/// Panics if the given value doesn't have a corresponding variant in `Value`. +fn into_value(val: Val) -> Value { + match val { + Val::I32(v) => Value::I32(v), + Val::I64(v) => Value::I64(v), + Val::F32(f_bits) => Value::F32(f_bits), + Val::F64(f_bits) => Value::F64(f_bits), + _ => panic!("Given value type is unsupported by substrate"), + } +} + +fn into_wasmtime_val(value: Value) -> wasmtime::Val { + match value { + Value::I32(v) => Val::I32(v), + Value::I64(v) => Val::I64(v), + Value::F32(f_bits) => Val::F32(f_bits), + Value::F64(f_bits) => Val::F64(f_bits), + } +} + +/// Attempt to convert a opaque panic payload to a string. +fn stringify_panic_payload(payload: Box) -> String { + match payload.downcast::<&'static str>() { + Ok(msg) => msg.to_string(), + Err(payload) => match payload.downcast::() { + Ok(msg) => *msg, + // At least we tried... + Err(_) => "Box".to_string(), + }, + } +} diff --git a/client/executor/wasmtime/src/instance_wrapper.rs b/client/executor/wasmtime/src/instance_wrapper.rs new file mode 100644 index 0000000000..013651cd7a --- /dev/null +++ b/client/executor/wasmtime/src/instance_wrapper.rs @@ -0,0 +1,276 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Defines data and logic needed for interaction with an WebAssembly instance of a substrate +//! runtime module. + +use crate::util; +use crate::imports::Imports; + +use sc_executor_common::error::{Error, Result}; +use sp_wasm_interface::{Pointer, WordSize, Value}; +use std::slice; +use std::marker; +use wasmtime::{Instance, Module, Memory, Table, Val}; + +/// Wrap the given WebAssembly Instance of a wasm module with Substrate-runtime. +/// +/// This struct is a handy wrapper around a wasmtime `Instance` that provides substrate specific +/// routines. +pub struct InstanceWrapper { + instance: Instance, + // The memory instance of the `intance`. + // + // It is important to make sure that we don't make any copies of this to make it easier to proof + // See `memory_as_slice` and `memory_as_slice_mut`. + memory: Memory, + table: Option, + // Make this struct explicitly !Send & !Sync. + _not_send_nor_sync: marker::PhantomData<*const ()>, +} + +impl InstanceWrapper { + /// Create a new instance wrapper from the given wasm module. + pub fn new(module: &Module, imports: &Imports, heap_pages: u32) -> Result { + let instance = Instance::new(module, &imports.externs) + .map_err(|e| Error::from(format!("cannot instantiate: {}", e)))?; + + let memory = match imports.memory_import_index { + Some(memory_idx) => { + imports.externs[memory_idx] + .memory() + .expect("only memory can be at the `memory_idx`; qed") + .clone() + } + None => { + let memory = get_linear_memory(&instance)?; + if !memory.grow(heap_pages).is_ok() { + return Err("failed top increase the linear memory size".into()); + } + memory + }, + }; + + Ok(Self { + table: get_table(&instance), + memory, + instance, + _not_send_nor_sync: marker::PhantomData, + }) + } + + /// Resolves a substrate entrypoint by the given name. + /// + /// An entrypoint must have a signature `(i32, i32) -> i64`, otherwise this function will return + /// an error. + pub fn resolve_entrypoint(&self, name: &str) -> Result { + // Resolve the requested method and verify that it has a proper signature. + let export = self + .instance + .get_export(name) + .ok_or_else(|| Error::from(format!("Exported method {} is not found", name)))?; + let entrypoint = export + .func() + .ok_or_else(|| Error::from(format!("Export {} is not a function", name)))?; + match (entrypoint.ty().params(), entrypoint.ty().results()) { + (&[wasmtime::ValType::I32, wasmtime::ValType::I32], &[wasmtime::ValType::I64]) => {} + _ => { + return Err(Error::from(format!( + "method {} have an unsupported signature", + name + ))) + } + } + Ok(entrypoint.clone()) + } + + /// Returns an indirect function table of this instance. + pub fn table(&self) -> Option<&Table> { + self.table.as_ref() + } + + /// Returns the byte size of the linear memory instance attached to this instance. + pub fn memory_size(&self) -> u32 { + self.memory.data_size() as u32 + } + + /// Reads `__heap_base: i32` global variable and returns it. + /// + /// If it doesn't exist, not a global or of not i32 type returns an error. + pub fn extract_heap_base(&self) -> Result { + let heap_base_export = self + .instance + .get_export("__heap_base") + .ok_or_else(|| Error::from("__heap_base is not found"))?; + + let heap_base_global = heap_base_export + .global() + .ok_or_else(|| Error::from("__heap_base is not a global"))?; + + let heap_base = heap_base_global + .get() + .i32() + .ok_or_else(|| Error::from("__heap_base is not a i32"))?; + + Ok(heap_base as u32) + } + + /// Get the value from a global with the given `name`. + pub fn get_global_val(&self, name: &str) -> Result> { + let global = match self.instance.get_export(name) { + Some(global) => global, + None => return Ok(None), + }; + + let global = global.global().ok_or_else(|| format!("`{}` is not a global", name))?; + + match global.get() { + Val::I32(val) => Ok(Some(Value::I32(val))), + Val::I64(val) => Ok(Some(Value::I64(val))), + Val::F32(val) => Ok(Some(Value::F32(val))), + Val::F64(val) => Ok(Some(Value::F64(val))), + _ => Err("Unknow value type".into()), + } + } +} + +/// Extract linear memory instance from the given instance. +fn get_linear_memory(instance: &Instance) -> Result { + let memory_export = instance + .get_export("memory") + .ok_or_else(|| Error::from("memory is not exported under `memory` name"))?; + + let memory = memory_export + .memory() + .ok_or_else(|| Error::from("the `memory` export should have memory type"))? + .clone(); + + Ok(memory) +} + +/// Extract the table from the given instance if any. +fn get_table(instance: &Instance) -> Option
{ + instance + .get_export("__indirect_function_table") + .and_then(|export| export.table()) + .cloned() +} + +/// Functions realted to memory. +impl InstanceWrapper { + /// Read data from a slice of memory into a destination buffer. + /// + /// Returns an error if the read would go out of the memory bounds. + pub fn read_memory_into(&self, address: Pointer, dest: &mut [u8]) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice(); + + let range = util::checked_range(address.into(), dest.len(), memory.len()) + .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; + dest.copy_from_slice(&memory[range]); + Ok(()) + } + } + + /// Write data to a slice of memory. + /// + /// Returns an error if the write would go out of the memory bounds. + pub fn write_memory_from(&self, address: Pointer, data: &[u8]) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + let range = util::checked_range(address.into(), data.len(), memory.len()) + .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; + &mut memory[range].copy_from_slice(data); + Ok(()) + } + } + + /// Allocate some memory of the given size. Returns pointer to the allocated memory region. + /// + /// Returns `Err` in case memory cannot be allocated. Refer to the allocator documentation + /// to get more details. + pub fn allocate( + &self, + allocator: &mut sp_allocator::FreeingBumpHeapAllocator, + size: WordSize, + ) -> Result> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + allocator.allocate(memory, size).map_err(Into::into) + } + } + + /// Deallocate the memory pointed by the given pointer. + /// + /// Returns `Err` in case the given memory region cannot be deallocated. + pub fn deallocate( + &self, + allocator: &mut sp_allocator::FreeingBumpHeapAllocator, + ptr: Pointer, + ) -> Result<()> { + unsafe { + // This should be safe since we don't grow up memory while caching this reference and + // we give up the reference before returning from this function. + let memory = self.memory_as_slice_mut(); + + allocator.deallocate(memory, ptr).map_err(Into::into) + } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// Wasmtime doesn't provide comprehensive documentation about the exact behavior of the data + /// pointer. If a dynamic style heap is used the base pointer of the heap can change. Since + /// growing, we cannot guarantee the lifetime of the returned slice reference. + unsafe fn memory_as_slice(&self) -> &[u8] { + let ptr = self.memory.data_ptr() as *const _; + let len = self.memory.data_size(); + + if len == 0 { + &[] + } else { + slice::from_raw_parts(ptr, len) + } + } + + /// Returns linear memory of the wasm instance as a slice. + /// + /// # Safety + /// + /// See `[memory_as_slice]`. In addition to those requirements, since a mutable reference is + /// returned it must be ensured that only one mutable and no shared references to memory exists + /// at the same time. + unsafe fn memory_as_slice_mut(&self) -> &mut [u8] { + let ptr = self.memory.data_ptr(); + let len = self.memory.data_size(); + + if len == 0 { + &mut [] + } else { + slice::from_raw_parts_mut(ptr, len) + } + } +} diff --git a/client/executor/wasmtime/src/lib.rs b/client/executor/wasmtime/src/lib.rs index 244fca8f84..8f4801e6da 100644 --- a/client/executor/wasmtime/src/lib.rs +++ b/client/executor/wasmtime/src/lib.rs @@ -16,10 +16,11 @@ ///! Defines a `WasmRuntime` that uses the Wasmtime JIT to execute. -mod function_executor; +mod host; mod runtime; -mod trampoline; +mod state_holder; +mod imports; +mod instance_wrapper; mod util; pub use runtime::create_instance; - diff --git a/client/executor/wasmtime/src/runtime.rs b/client/executor/wasmtime/src/runtime.rs index abf860667f..b99d334787 100644 --- a/client/executor/wasmtime/src/runtime.rs +++ b/client/executor/wasmtime/src/runtime.rs @@ -16,70 +16,51 @@ //! Defines the compiled Wasm runtime that uses Wasmtime internally. -use crate::function_executor::FunctionExecutorState; -use crate::trampoline::{EnvState, make_trampoline}; -use crate::util::{ - cranelift_ir_signature, - convert_parity_wasm_signature, - read_memory_into, - write_memory_from -}; +use crate::host::HostState; +use crate::imports::{resolve_imports, Imports}; +use crate::instance_wrapper::InstanceWrapper; +use crate::state_holder::StateHolder; use sc_executor_common::{ error::{Error, Result, WasmError}, wasm_runtime::WasmRuntime, }; -use sp_wasm_interface::{Pointer, WordSize, Function}; +use sp_allocator::FreeingBumpHeapAllocator; use sp_runtime_interface::unpack_ptr_and_len; +use sp_wasm_interface::{Function, Pointer, WordSize, Value}; +use wasmtime::{Config, Engine, Module, Store}; -use std::{cell::RefCell, collections::HashMap, convert::TryFrom, rc::Rc}; - -use cranelift_codegen::ir; -use cranelift_codegen::isa::TargetIsa; -use cranelift_entity::{EntityRef, PrimaryMap}; -use cranelift_frontend::FunctionBuilderContext; -use cranelift_wasm::{DefinedFuncIndex, MemoryIndex}; -use wasmtime_environ::{Module, translate_signature}; -use wasmtime_jit::{ - ActionOutcome, CodeMemory, CompilationStrategy, CompiledModule, Compiler, Context, RuntimeValue, -}; -use wasmtime_runtime::{Export, Imports, InstanceHandle, VMFunctionBody}; - -/// TODO: We should remove this in https://github.com/paritytech/substrate/pull/4686 -/// Currently there is no way to extract this with wasmtime. -const INITIAL_HEAP_PAGES: u32 = 17; - -/// A `WasmRuntime` implementation using the Wasmtime JIT to compile the runtime module to native +/// A `WasmRuntime` implementation using wasmtime to compile the runtime module to machine code /// and execute the compiled code. pub struct WasmtimeRuntime { - module: CompiledModule, - context: Context, + module: Module, + imports: Imports, + state_holder: StateHolder, heap_pages: u32, - /// The host functions registered for this instance. host_functions: Vec<&'static dyn Function>, - /// The index of the memory in the module. - memory_index: MemoryIndex, } impl WasmRuntime for WasmtimeRuntime { - fn update_heap_pages(&mut self, heap_pages: u64) -> bool { - self.heap_pages as u64 == heap_pages - } - fn host_functions(&self) -> &[&'static dyn Function] { &self.host_functions } fn call(&mut self, method: &str, data: &[u8]) -> Result> { call_method( - &mut self.context, - &mut self.module, + &self.module, + &mut self.imports, + &self.state_holder, method, data, - self.memory_index, self.heap_pages, ) } + + fn get_global_val(&self, name: &str) -> Result> { + // Yeah, there is no better way currently :( + InstanceWrapper::new(&self.module, &self.imports, self.heap_pages)? + .get_global_val(name) + } } /// Create a new `WasmtimeRuntime` given the code. This function performs translation from Wasm to @@ -90,445 +71,105 @@ pub fn create_instance( host_functions: Vec<&'static dyn Function>, allow_missing_func_imports: bool, ) -> std::result::Result { - let heap_pages = u32::try_from(heap_pages) - .map_err(|e| - WasmError::Other(format!("Heap pages can not be converted into `u32`: {:?}", e)) - )?; - - let (compiled_module, context, memory_index) = create_compiled_unit( - code, + // Create the engine, store and finally the module from the given code. + let mut config = Config::new(); + config.cranelift_opt_level(wasmtime::OptLevel::SpeedAndSize); + + let engine = Engine::new(&config); + let store = Store::new(&engine); + let module = Module::new(&store, code) + .map_err(|e| WasmError::Other(format!("cannot create module: {}", e)))?; + + let state_holder = StateHolder::empty(); + + // Scan all imports, find the matching host functions, and create stubs that adapt arguments + // and results. + let imports = resolve_imports( + &state_holder, + &module, &host_functions, - heap_pages, + heap_pages as u32, allow_missing_func_imports, )?; - let module = compiled_module.module_ref(); - if !module.is_imported_memory(memory_index) { - // Inspect the module for the min and max memory sizes. - let (min_memory_size, max_memory_size) = { - let memory_plan = module.memory_plans - .get(memory_index) - .ok_or_else(|| WasmError::InvalidMemory)?; - (memory_plan.memory.minimum, memory_plan.memory.maximum) - }; - - // Check that heap_pages is within the allowed range. - let max_heap_pages = max_memory_size.map(|max| max.saturating_sub(min_memory_size)); - - if max_heap_pages.map(|m| heap_pages > m).unwrap_or(false) { - return Err(WasmError::InvalidHeapPages) - } - } - Ok(WasmtimeRuntime { - module: compiled_module, - context, - heap_pages, + module, + imports, + state_holder, + heap_pages: heap_pages as u32, host_functions, - memory_index, }) } -#[derive(Debug)] -struct MissingFunction { - name: String, - sig: cranelift_codegen::ir::Signature, -} - -#[derive(Debug)] -struct MissingFunctionStubs { - stubs: HashMap>, -} - -impl MissingFunctionStubs { - fn new() -> Self { - Self { - stubs: HashMap::new(), - } - } - - fn insert(&mut self, module: String, name: String, sig: cranelift_codegen::ir::Signature) { - self.stubs.entry(module).or_insert_with(Vec::new).push(MissingFunction { - name, - sig, - }); - } -} - -fn scan_missing_functions( - code: &[u8], - host_functions: &[&'static dyn Function], -) -> std::result::Result { - let isa = target_isa()?; - let call_conv = isa.default_call_conv(); - - let module = parity_wasm::elements::Module::from_bytes(code) - .map_err(|e| WasmError::Other(format!("cannot deserialize error: {}", e)))?; - - let types = module.type_section().map(|ts| ts.types()).unwrap_or(&[]); - let import_entries = module - .import_section() - .map(|is| is.entries()) - .unwrap_or(&[]); - - let mut missing_functions = MissingFunctionStubs::new(); - for import_entry in import_entries { - let func_ty = match import_entry.external() { - parity_wasm::elements::External::Function(func_ty_idx) => { - let parity_wasm::elements::Type::Function(ref func_ty) = - types.get(*func_ty_idx as usize).ok_or_else(|| { - WasmError::Other(format!("corrupted module, type out of bounds")) - })?; - func_ty - } - _ => { - // We are looking only for missing **functions** here. Any other items, be they - // missing or not, will be handled at the resolution stage later. - continue; - } - }; - let signature = convert_parity_wasm_signature(func_ty); - - if import_entry.module() == "env" { - if let Some(hf) = host_functions - .iter() - .find(|hf| hf.name() == import_entry.field()) - { - if signature == hf.signature() { - continue; - } - } - } - - // This function is either not from the env module, or doesn't have a corresponding host - // function, or the signature is not matching. Add it to the list. - let sig = cranelift_ir_signature(signature, &call_conv); - - missing_functions.insert( - import_entry.module().to_string(), - import_entry.field().to_string(), - sig, - ); - } - - Ok(missing_functions) -} - -fn create_compiled_unit( - code: &[u8], - host_functions: &[&'static dyn Function], - heap_pages: u32, - allow_missing_func_imports: bool, -) -> std::result::Result<(CompiledModule, Context, MemoryIndex), WasmError> { - let compilation_strategy = CompilationStrategy::Cranelift; - - let compiler = new_compiler(compilation_strategy)?; - let mut context = Context::new(Box::new(compiler)); - - // Enable/disable producing of debug info. - context.set_debug_info(false); - - // Instantiate and link the env module. - let global_exports = context.get_global_exports(); - let compiler = new_compiler(compilation_strategy)?; - - let mut missing_functions_stubs = if allow_missing_func_imports { - scan_missing_functions(code, host_functions)? - } else { - // If there are in fact missing functions they will be detected at the instantiation time - // and the module will be rejected. - MissingFunctionStubs::new() - }; - - let env_missing_functions = missing_functions_stubs.stubs - .remove("env") - .unwrap_or_else(|| Vec::new()); - - let (module, memory_index) = instantiate_env_module( - global_exports, - compiler, - host_functions, - heap_pages, - env_missing_functions, - true, - )?; - - context.name_instance("env".to_owned(), module); - - for (module, missing_functions_stubs) in missing_functions_stubs.stubs { - let compiler = new_compiler(compilation_strategy)?; - let global_exports = context.get_global_exports(); - let instance = instantiate_env_module( - global_exports, - compiler, - &[], - heap_pages, - missing_functions_stubs, - false, - )?.0; - context.name_instance(module, instance); - } - - // Compile the wasm module. - let module = context.compile_module(&code) - .map_err(|e| WasmError::Other(format!("module compile error: {}", e)))?; - - Ok((module, context, memory_index.expect("Memory is added on request; qed"))) -} - /// Call a function inside a precompiled Wasm module. fn call_method( - context: &mut Context, - module: &mut CompiledModule, + module: &Module, + imports: &mut Imports, + state_holder: &StateHolder, method: &str, data: &[u8], - memory_index: MemoryIndex, heap_pages: u32, ) -> Result> { - let is_imported_memory = module.module().is_imported_memory(memory_index); - // Old exports get clobbered in `InstanceHandle::new` if we don't explicitly remove them first. - // - // The global exports mechanism is temporary in Wasmtime and expected to be removed. - // https://github.com/CraneStation/wasmtime/issues/332 - clear_globals(&mut *context.get_global_exports().borrow_mut(), is_imported_memory); - - let mut instance = module.instantiate().map_err(|e| Error::Other(e.to_string()))?; - - if !is_imported_memory { - grow_memory(&mut instance, heap_pages)?; - } - - // Initialize the function executor state. - let heap_base = get_heap_base(&instance)?; - let executor_state = FunctionExecutorState::new(heap_base); - reset_env_state_and_take_trap(context, Some(executor_state))?; - - // Write the input data into guest memory. - let (data_ptr, data_len) = inject_input_data(context, &mut instance, data, memory_index)?; - let args = [RuntimeValue::I32(u32::from(data_ptr) as i32), RuntimeValue::I32(data_len as i32)]; - - // Invoke the function in the runtime. - let outcome = context - .invoke(&mut instance, method, &args[..]) - .map_err(|e| Error::Other(format!("error calling runtime: {}", e)))?; - let trap_error = reset_env_state_and_take_trap(context, None)?; - let (output_ptr, output_len) = match outcome { - ActionOutcome::Returned { values } => match values.as_slice() { - [RuntimeValue::I64(retval)] => unpack_ptr_and_len(*retval as u64), - _ => return Err(Error::InvalidReturn), - } - ActionOutcome::Trapped { message } => return Err(trap_error.unwrap_or_else( - || format!("Wasm execution trapped: {}", message).into() - )), - }; - - // Read the output data from guest memory. - let mut output = vec![0; output_len as usize]; - let memory = get_memory_mut(&mut instance, memory_index)?; - read_memory_into(memory, Pointer::new(output_ptr), &mut output)?; - Ok(output) -} - -/// The implementation is based on wasmtime_wasi::instantiate_wasi. -fn instantiate_env_module( - global_exports: Rc>>>, - compiler: Compiler, - host_functions: &[&'static dyn Function], - heap_pages: u32, - missing_functions_stubs: Vec, - add_memory: bool, -) -> std::result::Result<(InstanceHandle, Option), WasmError> { - let isa = target_isa()?; - let pointer_type = isa.pointer_type(); - let call_conv = isa.default_call_conv(); - - let mut fn_builder_ctx = FunctionBuilderContext::new(); - let mut module = Module::new(); - let mut finished_functions = >::new(); - let mut code_memory = CodeMemory::new(); - - for function in host_functions { - let sig = translate_signature( - cranelift_ir_signature(function.signature(), &call_conv), - pointer_type, - ); - let sig_id = module.signatures.push(sig.clone()); - let func_id = module.functions.push(sig_id); - module - .exports - .insert(function.name().to_string(), wasmtime_environ::Export::Function(func_id)); - - let trampoline = make_trampoline( - isa.as_ref(), - &mut code_memory, - &mut fn_builder_ctx, - func_id.index() as u32, - &sig, - )?; - finished_functions.push(trampoline); - } - - for MissingFunction { name, sig } in missing_functions_stubs { - let sig = translate_signature( - sig, - pointer_type, - ); - let sig_id = module.signatures.push(sig.clone()); - let func_id = module.functions.push(sig_id); - module - .exports - .insert(name, wasmtime_environ::Export::Function(func_id)); - let trampoline = make_trampoline( - isa.as_ref(), - &mut code_memory, - &mut fn_builder_ctx, - func_id.index() as u32, - &sig, - )?; - finished_functions.push(trampoline); - } - - code_memory.publish(); - - let memory_id = if add_memory { - let memory = cranelift_wasm::Memory { - minimum: heap_pages + INITIAL_HEAP_PAGES, - maximum: Some(heap_pages + INITIAL_HEAP_PAGES), - shared: false, - }; - let memory_plan = wasmtime_environ::MemoryPlan::for_memory(memory, &Default::default()); - - let memory_id = module.memory_plans.push(memory_plan); - module.exports.insert("memory".into(), wasmtime_environ::Export::Memory(memory_id)); - - Some(memory_id) - } else { - None - }; + let instance_wrapper = InstanceWrapper::new(module, imports, heap_pages)?; + let entrypoint = instance_wrapper.resolve_entrypoint(method)?; + let heap_base = instance_wrapper.extract_heap_base()?; + let allocator = FreeingBumpHeapAllocator::new(heap_base); - let imports = Imports::none(); - let data_initializers = Vec::new(); - let signatures = PrimaryMap::new(); - let env_state = EnvState::new(code_memory, compiler, host_functions); - - let result = InstanceHandle::new( - Rc::new(module), - global_exports, - finished_functions.into_boxed_slice(), - imports, - &data_initializers, - signatures.into_boxed_slice(), - None, - Box::new(env_state), - ); - - result - .map_err(|e| WasmError::Other(format!("cannot instantiate env: {}", e))) - .map(|r| (r, memory_id)) -} - -/// Build a new TargetIsa for the host machine. -fn target_isa() -> std::result::Result, WasmError> { - let isa_builder = cranelift_native::builder() - .map_err(|e| WasmError::Other(format!("missing compiler support: {}", e)))?; - let flag_builder = cranelift_codegen::settings::builder(); - Ok(isa_builder.finish(cranelift_codegen::settings::Flags::new(flag_builder))) -} - -fn new_compiler(strategy: CompilationStrategy) -> std::result::Result { - let isa = target_isa()?; - Ok(Compiler::new(isa, strategy)) + perform_call(data, state_holder, instance_wrapper, entrypoint, allocator) } -fn clear_globals(global_exports: &mut HashMap>, is_imported_memory: bool) { - // When memory is imported, we can not delete the global export. - if !is_imported_memory { - global_exports.remove("memory"); - } - global_exports.remove("__heap_base"); - global_exports.remove("__indirect_function_table"); -} - -fn grow_memory(instance: &mut InstanceHandle, pages: u32) -> Result<()> { - // This is safe to wrap in an unsafe block as: - // - The result of the `lookup_immutable` call is not mutated - // - The definition pointer is returned by a lookup on a valid instance - let memory_index = unsafe { - match instance.lookup_immutable("memory") { - Some(Export::Memory { definition, vmctx: _, memory: _ }) => - instance.memory_index(&*definition), - _ => return Err(Error::InvalidMemoryReference), +fn perform_call( + data: &[u8], + state_holder: &StateHolder, + instance_wrapper: InstanceWrapper, + entrypoint: wasmtime::Func, + mut allocator: FreeingBumpHeapAllocator, +) -> Result> { + let (data_ptr, data_len) = inject_input_data(&instance_wrapper, &mut allocator, data)?; + + let host_state = HostState::new(allocator, instance_wrapper); + let (ret, host_state) = state_holder.with_initialized_state(host_state, || { + match entrypoint.call(&[ + wasmtime::Val::I32(u32::from(data_ptr) as i32), + wasmtime::Val::I32(u32::from(data_len) as i32), + ]) { + Ok(results) => { + let retval = results[0].unwrap_i64() as u64; + Ok(unpack_ptr_and_len(retval)) + } + Err(trap) => { + return Err(Error::from(format!( + "Wasm execution trapped: {}", + trap.message() + ))); + } } - }; - instance.memory_grow(memory_index, pages) - .map(|_| ()) - .ok_or_else(|| "requested heap_pages would exceed maximum memory size".into()) -} + }); + let (output_ptr, output_len) = ret?; -fn get_env_state(context: &mut Context) -> Result<&mut EnvState> { - let env_instance = context.get_instance("env") - .map_err(|err| format!("cannot find \"env\" module: {}", err))?; - env_instance - .host_state() - .downcast_mut::() - .ok_or_else(|| "cannot get \"env\" module host state".into()) -} + let instance = host_state.into_instance(); + let output = extract_output_data(&instance, output_ptr, output_len)?; -fn reset_env_state_and_take_trap( - context: &mut Context, - executor_state: Option, -) -> Result> -{ - let env_state = get_env_state(context)?; - env_state.executor_state = executor_state; - Ok(env_state.take_trap()) + Ok(output) } fn inject_input_data( - context: &mut Context, - instance: &mut InstanceHandle, + instance: &InstanceWrapper, + allocator: &mut FreeingBumpHeapAllocator, data: &[u8], - memory_index: MemoryIndex, ) -> Result<(Pointer, WordSize)> { - let env_state = get_env_state(context)?; - let executor_state = env_state.executor_state - .as_mut() - .ok_or_else(|| "cannot get \"env\" module executor state")?; - - let memory = get_memory_mut(instance, memory_index)?; - let data_len = data.len() as WordSize; - let data_ptr = executor_state.heap().allocate(memory, data_len)?; - write_memory_from(memory, data_ptr, data)?; + let data_ptr = instance.allocate(allocator, data_len)?; + instance.write_memory_from(data_ptr, data)?; Ok((data_ptr, data_len)) } -fn get_memory_mut(instance: &mut InstanceHandle, memory_index: MemoryIndex) -> Result<&mut [u8]> { - match instance.lookup_by_declaration(&wasmtime_environ::Export::Memory(memory_index)) { - // This is safe to wrap in an unsafe block as: - // - The definition pointer is returned by a lookup on a valid instance and thus points to - // a valid memory definition - Export::Memory { definition, vmctx: _, memory: _ } => unsafe { - Ok(std::slice::from_raw_parts_mut( - (*definition).base, - (*definition).current_length, - )) - }, - _ => Err(Error::InvalidMemoryReference), - } -} - -fn get_heap_base(instance: &InstanceHandle) -> Result { - // This is safe to wrap in an unsafe block as: - // - The result of the `lookup_immutable` call is not mutated - // - The definition pointer is returned by a lookup on a valid instance - // - The defined value is checked to be an I32, which can be read safely as a u32 - unsafe { - match instance.lookup_immutable("__heap_base") { - Some(Export::Global { definition, vmctx: _, global }) - if global.ty == ir::types::I32 => - Ok(*(*definition).as_u32()), - _ => return Err(Error::HeapBaseNotFoundOrInvalid), - } - } +fn extract_output_data( + instance: &InstanceWrapper, + output_ptr: u32, + output_len: u32, +) -> Result> { + let mut output = vec![0; output_len as usize]; + instance.read_memory_into(Pointer::new(output_ptr), &mut output)?; + Ok(output) } diff --git a/client/executor/wasmtime/src/state_holder.rs b/client/executor/wasmtime/src/state_holder.rs new file mode 100644 index 0000000000..57564ed3ec --- /dev/null +++ b/client/executor/wasmtime/src/state_holder.rs @@ -0,0 +1,77 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::host::{HostContext, HostState}; +use std::cell::RefCell; +use std::rc::Rc; + +/// A common place to store a reference to the `HostState`. +/// +/// This structure is passed into each host function handler and retained in the implementation of +/// `WasmRuntime`. Whenever a call into a runtime method is initiated, the host state is populated +/// with the state for that runtime method call. +/// +/// During the execution of the runtime method call, wasm can call imported host functions. When +/// that happens the host function handler gets a `HostContext` (obtainable through having a +/// `HostState` reference). +#[derive(Clone)] +pub struct StateHolder { + // This is `Some` only during a call. + state: Rc>>, +} + +impl StateHolder { + /// Create a placeholder `StateHolder`. + pub fn empty() -> StateHolder { + StateHolder { + state: Rc::new(RefCell::new(None)), + } + } + + /// Provide `HostState` for the runtime method call and execute the given function `f`. + /// + /// During the execution of the provided function `with_context` will be callable. + pub fn with_initialized_state(&self, state: HostState, f: F) -> (R, HostState) + where + F: FnOnce() -> R, + { + *self.state.borrow_mut() = Some(state); + + let ret = f(); + let state = self + .state + .borrow_mut() + .take() + .expect("cannot be None since was just assigned; qed"); + + (ret, state) + } + + /// Create a `HostContext` from the contained `HostState` and execute the given function `f`. + /// + /// This function is only callable within closure passed to `init_state`. Otherwise, the passed + /// context will be `None`. + pub fn with_context(&self, f: F) -> R + where + F: FnOnce(Option) -> R, + { + let state = self.state.borrow(); + match *state { + Some(ref state) => f(Some(state.materialize())), + None => f(None), + } + } +} diff --git a/client/executor/wasmtime/src/trampoline.rs b/client/executor/wasmtime/src/trampoline.rs deleted file mode 100644 index 8a21477609..0000000000 --- a/client/executor/wasmtime/src/trampoline.rs +++ /dev/null @@ -1,361 +0,0 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! The trampoline is the dynamically generated entry point to a runtime host call. -//! -//! This code is based on and large parts are copied from wasmtime's -//! wasmtime-api/src/trampoline/func.rs. - -use crate::function_executor::{FunctionExecutorState, FunctionExecutor}; -use sc_executor_common::error::{Error, WasmError}; - -use cranelift_codegen::{Context, binemit, ir, isa}; -use cranelift_codegen::ir::{InstBuilder, StackSlotData, StackSlotKind, TrapCode}; -use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext}; -use cranelift_codegen::print_errors::pretty_error; -use wasmtime_jit::{CodeMemory, Compiler}; -use wasmtime_environ::CompiledFunction; -use wasmtime_runtime::{VMContext, VMFunctionBody}; -use sp_wasm_interface::{Function, Value, ValueType}; -use std::{cmp, panic::{self, AssertUnwindSafe}, ptr}; - -const CALL_SUCCESS: u32 = 0; -const CALL_FAILED_WITH_ERROR: u32 = 1; -const CALL_WITH_BAD_HOST_STATE: u32 = 2; - -/// A code to trap with that indicates a host call error. -const TRAP_USER_CODE: u16 = 0; - -/// The only Wasm types allowed in host function signatures (I32, I64, F32, F64) are all -/// represented in at most 8 bytes. -const MAX_WASM_TYPE_SIZE: usize = 8; - -/// The top-level host state of the "env" module. This state is used by the trampoline function to -/// construct a `FunctionExecutor` which can execute the host call. -pub struct EnvState { - host_functions: Vec<&'static dyn Function>, - compiler: Compiler, - // The code memory must be kept around on the state to prevent it from being dropped. - #[allow(dead_code)] - code_memory: CodeMemory, - trap: Option, - /// The executor state stored across host calls during a single Wasm runtime call. - /// During a runtime call, this MUST be `Some`. - pub executor_state: Option, -} - -impl EnvState { - /// Construct a new `EnvState` which owns the given code memory. - pub fn new( - code_memory: CodeMemory, - compiler: Compiler, - host_functions: &[&'static dyn Function], - ) -> Self { - EnvState { - trap: None, - compiler, - code_memory, - executor_state: None, - host_functions: host_functions.to_vec(), - } - } - - /// Resets the trap error to None and returns the current value. - pub fn take_trap(&mut self) -> Option { - self.trap.take() - } -} - -/// This is called by the dynamically generated trampoline taking the function index and reference -/// to the call arguments on the stack as arguments. Returns zero on success and a non-zero value -/// on failure. -unsafe extern "C" fn stub_fn(vmctx: *mut VMContext, func_index: u32, values_vec: *mut i64) -> u32 { - if let Some(state) = (*vmctx).host_state().downcast_mut::() { - match stub_fn_inner( - vmctx, - &state.host_functions, - &mut state.compiler, - state.executor_state.as_mut(), - func_index, - values_vec, - ) { - Ok(()) => CALL_SUCCESS, - Err(err) => { - state.trap = Some(err); - CALL_FAILED_WITH_ERROR - } - } - } else { - // Well, we can't even set a trap message, so we'll just exit without one. - CALL_WITH_BAD_HOST_STATE - } -} - -/// Implements most of the logic in `stub_fn` but returning a `Result` instead of an integer error -/// for the sake of readability. -unsafe fn stub_fn_inner( - vmctx: *mut VMContext, - externals: &[&dyn Function], - compiler: &mut Compiler, - executor_state: Option<&mut FunctionExecutorState>, - func_index: u32, - values_vec: *mut i64, -) -> Result<(), Error> { - let func = externals.get(func_index as usize) - .ok_or_else(|| format!("call to undefined external function with index {}", func_index))?; - let executor_state = executor_state - .ok_or_else(|| "executor state is None during call to external function")?; - - // Build the external function context. - let mut context = FunctionExecutor::new(vmctx, compiler, executor_state)?; - let mut context = AssertUnwindSafe(&mut context); - - // Execute and write output back to the stack. - let return_val = panic::catch_unwind(move || { - let signature = func.signature(); - - // Read the arguments from the stack. - let mut args = signature.args.iter() - .enumerate() - .map(|(i, ¶m_type)| read_value_from(values_vec.offset(i as isize), param_type)); - - func.execute(&mut **context, &mut args) - }); - - match return_val { - Ok(ret_val) => { - if let Some(val) = ret_val - .map_err(|e| Error::FunctionExecution(func.name().to_string(), e))? { - write_value_to(values_vec, val); - } - - Ok(()) - }, - Err(e) => { - let message = if let Some(err) = e.downcast_ref::() { - err.to_string() - } else if let Some(err) = e.downcast_ref::<&str>() { - err.to_string() - } else { - "Panicked without any further information!".into() - }; - - Err(Error::FunctionExecution(func.name().to_string(), message)) - } - } -} - -/// Create a trampoline for invoking a host function. -/// -/// The trampoline is a dynamically generated entry point to a runtime host call. The function is -/// generated by manually constructing Cranelift IR and using the Cranelift compiler. The -/// trampoline embeds the function index as a constant and delegates to a stub function in Rust, -/// which takes the function index and a memory reference to the stack arguments and return value -/// slots. -/// -/// This code is of modified copy of wasmtime's wasmtime-api/src/trampoline/func.rs. -pub fn make_trampoline( - isa: &dyn isa::TargetIsa, - code_memory: &mut CodeMemory, - fn_builder_ctx: &mut FunctionBuilderContext, - func_index: u32, - signature: &ir::Signature, -) -> Result<*const VMFunctionBody, WasmError> { - // Mostly reverse copy of the similar method from wasmtime's - // wasmtime-jit/src/compiler.rs. - let pointer_type = isa.pointer_type(); - let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Ensure that the first parameter of the generated function is the `VMContext` pointer. - assert_eq!( - signature.params[0], - ir::AbiParam::special(pointer_type, ir::ArgumentPurpose::VMContext) - ); - - // Add the `vmctx` parameter. - stub_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the `func_index` parameter. - stub_sig.params.push(ir::AbiParam::new(ir::types::I32)); - - // Add the `values_vec` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); - - // Add error/trap return. - stub_sig.returns.push(ir::AbiParam::new(ir::types::I32)); - - // Each parameter and return value gets a 64-bit (8-byte) wide slot on the stack, as that is - // large enough to fit all Wasm primitive types that can be used in host function signatures. - // The `VMContext` pointer, which is a parameter of the function signature, is excluded as it - // is passed directly to the stub function rather than being looked up on the caller stack from - // the `values_vec` pointer. - let values_vec_len = cmp::max(signature.params.len() - 1, signature.returns.len()); - let values_vec_size = (MAX_WASM_TYPE_SIZE * values_vec_len) as u32; - - let mut context = Context::new(); - context.func = - ir::Function::with_name_signature(ir::ExternalName::user(0, 0), signature.clone()); - - let ss = context.func.create_stack_slot(StackSlotData::new( - StackSlotKind::ExplicitSlot, - values_vec_size, - )); - - { - let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); - let block0 = builder.create_ebb(); - - builder.append_ebb_params_for_function_params(block0); - builder.switch_to_block(block0); - builder.seal_block(block0); - - let values_vec_ptr_val = builder.ins().stack_addr(pointer_type, ss, 0); - let mflags = ir::MemFlags::trusted(); - for i in 1..signature.params.len() { - let val = builder.func.dfg.ebb_params(block0)[i]; - builder.ins().store( - mflags, - val, - values_vec_ptr_val, - ((i - 1) * MAX_WASM_TYPE_SIZE) as i32, - ); - } - - let vmctx_ptr_val = builder.func.dfg.ebb_params(block0)[0]; - let func_index_val = builder.ins().iconst(ir::types::I32, func_index as i64); - - let callee_args = vec![vmctx_ptr_val, func_index_val, values_vec_ptr_val]; - - let new_sig = builder.import_signature(stub_sig.clone()); - - let callee_value = builder - .ins() - .iconst(pointer_type, stub_fn as *const VMFunctionBody as i64); - let call = builder - .ins() - .call_indirect(new_sig, callee_value, &callee_args); - - let call_result = builder.func.dfg.inst_results(call)[0]; - builder.ins().trapnz(call_result, TrapCode::User(TRAP_USER_CODE)); - - let mflags = ir::MemFlags::trusted(); - let mut results = Vec::new(); - for (i, r) in signature.returns.iter().enumerate() { - let load = builder.ins().load( - r.value_type, - mflags, - values_vec_ptr_val, - (i * MAX_WASM_TYPE_SIZE) as i32, - ); - results.push(load); - } - builder.ins().return_(&results); - builder.finalize() - } - - let mut code_buf: Vec = Vec::new(); - let mut reloc_sink = RelocSink; - let mut trap_sink = binemit::NullTrapSink {}; - let mut stackmap_sink = binemit::NullStackmapSink {}; - context - .compile_and_emit( - isa, - &mut code_buf, - &mut reloc_sink, - &mut trap_sink, - &mut stackmap_sink, - ) - .map_err(|e| { - WasmError::Instantiation(format!( - "failed to compile trampoline: {}", - pretty_error(&context.func, Some(isa), e) - )) - })?; - - let mut unwind_info = Vec::new(); - context.emit_unwind_info(isa, &mut unwind_info); - - let func_ref = code_memory - .allocate_for_function(&CompiledFunction { - body: code_buf, - jt_offsets: context.func.jt_offsets, - unwind_info, - }) - .map_err(|e| WasmError::Instantiation(format!("failed to allocate code memory: {}", e)))?; - - Ok(func_ref.as_ptr()) -} - -/// We don't expect trampoline compilation to produce any relocations, so -/// this `RelocSink` just asserts that it doesn't recieve any. -struct RelocSink; - -impl binemit::RelocSink for RelocSink { - fn reloc_ebb( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _ebb_offset: binemit::CodeOffset, - ) { - panic!("trampoline compilation should not produce ebb relocs"); - } - fn reloc_external( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _name: &ir::ExternalName, - _addend: binemit::Addend, - ) { - panic!("trampoline compilation should not produce external symbol relocs"); - } - fn reloc_constant( - &mut self, - _code_offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _constant_offset: ir::ConstantOffset, - ) { - panic!("trampoline compilation should not produce constant relocs"); - } - fn reloc_jt( - &mut self, - _offset: binemit::CodeOffset, - _reloc: binemit::Reloc, - _jt: ir::JumpTable, - ) { - panic!("trampoline compilation should not produce jump table relocs"); - } -} - -unsafe fn write_value_to(p: *mut i64, val: Value) { - match val { - Value::I32(i) => ptr::write(p as *mut i32, i), - Value::I64(i) => ptr::write(p as *mut i64, i), - Value::F32(u) => ptr::write(p as *mut u32, u), - Value::F64(u) => ptr::write(p as *mut u64, u), - } -} - -unsafe fn read_value_from(p: *const i64, ty: ValueType) -> Value { - match ty { - ValueType::I32 => Value::I32(ptr::read(p as *const i32)), - ValueType::I64 => Value::I64(ptr::read(p as *const i64)), - ValueType::F32 => Value::F32(ptr::read(p as *const u32)), - ValueType::F64 => Value::F64(ptr::read(p as *const u64)), - } -} diff --git a/client/executor/wasmtime/src/util.rs b/client/executor/wasmtime/src/util.rs index 77fb8af386..d2de95d4cc 100644 --- a/client/executor/wasmtime/src/util.rs +++ b/client/executor/wasmtime/src/util.rs @@ -1,4 +1,4 @@ -// Copyright 2019-2020 Parity Technologies (UK) Ltd. +// Copyright 2020 Parity Technologies (UK) Ltd. // This file is part of Substrate. // Substrate is free software: you can redistribute it and/or modify @@ -14,31 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use sc_executor_common::error::{Error, Result}; - -use cranelift_codegen::{ir, isa}; use std::ops::Range; -use sp_wasm_interface::{Pointer, Signature, ValueType}; - -/// Read data from a slice of memory into a destination buffer. -/// -/// Returns an error if the read would go out of the memory bounds. -pub fn read_memory_into(memory: &[u8], address: Pointer, dest: &mut [u8]) -> Result<()> { - let range = checked_range(address.into(), dest.len(), memory.len()) - .ok_or_else(|| Error::Other("memory read is out of bounds".into()))?; - dest.copy_from_slice(&memory[range]); - Ok(()) -} - -/// Write data to a slice of memory. -/// -/// Returns an error if the write would go out of the memory bounds. -pub fn write_memory_from(memory: &mut [u8], address: Pointer, data: &[u8]) -> Result<()> { - let range = checked_range(address.into(), data.len(), memory.len()) - .ok_or_else(|| Error::Other("memory write is out of bounds".into()))?; - &mut memory[range].copy_from_slice(data); - Ok(()) -} /// Construct a range from an offset to a data length after the offset. /// Returns None if the end of the range would exceed some maximum offset. @@ -50,82 +26,3 @@ pub fn checked_range(offset: usize, len: usize, max: usize) -> Option Signature { - fn convert_value_type(val_ty: parity_wasm::elements::ValueType) -> ValueType { - use parity_wasm::elements::ValueType::*; - match val_ty { - I32 => ValueType::I32, - I64 => ValueType::I64, - F32 => ValueType::F32, - F64 => ValueType::F64, - } - } - - Signature::new( - func_ty.params().iter().cloned().map(convert_value_type).collect::>(), - func_ty.return_type().map(convert_value_type), - ) -} - -/// Convert a wasm_interface Signature into a cranelift_codegen Signature. -pub fn cranelift_ir_signature(signature: Signature, call_conv: &isa::CallConv) -> ir::Signature { - ir::Signature { - params: signature.args.iter() - .map(cranelift_ir_type) - .map(ir::AbiParam::new) - .collect(), - returns: signature.return_value.iter() - .map(cranelift_ir_type) - .map(ir::AbiParam::new) - .collect(), - call_conv: call_conv.clone(), - } -} - -/// Convert a wasm_interface ValueType into a cranelift_codegen Type. -pub fn cranelift_ir_type(value_type: &ValueType) -> ir::types::Type { - match value_type { - ValueType::I32 => ir::types::I32, - ValueType::I64 => ir::types::I64, - ValueType::F32 => ir::types::F32, - ValueType::F64 => ir::types::F64, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use assert_matches::assert_matches; - - #[test] - fn test_read_memory_into() { - let mut memory = [0; 20]; - let mut dest = [0; 5]; - - &mut memory[15..20].copy_from_slice(b"hello"); - - read_memory_into(&memory[..], Pointer::new(15), &mut dest[..]).unwrap(); - - // Test that out of bounds read fails. - assert_matches!( - read_memory_into(&memory[..], Pointer::new(16), &mut dest[..]), - Err(Error::Other(_)) - ) - } - - #[test] - fn test_write_memory_from() { - let mut memory = [0; 20]; - let data = b"hello"; - - write_memory_from(&mut memory[..], Pointer::new(15), data).unwrap(); - - // Test that out of bounds write fails. - assert_matches!( - write_memory_from(&mut memory[..], Pointer::new(16), data), - Err(Error::Other(_)) - ) - } -} diff --git a/client/finality-grandpa/Cargo.toml b/client/finality-grandpa/Cargo.toml index 1249bff751..e96792258a 100644 --- a/client/finality-grandpa/Cargo.toml +++ b/client/finality-grandpa/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } futures = "0.3.1" -futures-timer = "2.0.2" +futures-timer = "3.0.1" log = "0.4.8" parking_lot = "0.10.0" rand = "0.7.2" diff --git a/client/finality-grandpa/src/communication/mod.rs b/client/finality-grandpa/src/communication/mod.rs index 227ecaa370..540923c1b1 100644 --- a/client/finality-grandpa/src/communication/mod.rs +++ b/client/finality-grandpa/src/communication/mod.rs @@ -178,7 +178,6 @@ impl> NetworkBridge { service: N, config: crate::Config, set_state: crate::environment::SharedVoterSetState, - executor: &impl futures::task::Spawn, ) -> Self { let (validator, report_stream) = GossipValidator::new( config, @@ -186,7 +185,7 @@ impl> NetworkBridge { ); let validator = Arc::new(validator); - let gossip_engine = GossipEngine::new(service.clone(), executor, GRANDPA_ENGINE_ID, validator.clone()); + let gossip_engine = GossipEngine::new(service.clone(), GRANDPA_ENGINE_ID, validator.clone()); { // register all previous votes with the gossip service so that they're @@ -374,10 +373,9 @@ impl> NetworkBridge { |to, neighbor| self.neighbor_sender.send(to, neighbor), ); - let service = self.gossip_engine.clone(); let topic = global_topic::(set_id.0); let incoming = incoming_global( - service, + self.gossip_engine.clone(), topic, voters, self.validator.clone(), @@ -419,7 +417,7 @@ impl> NetworkBridge { impl> Future for NetworkBridge { type Output = Result<(), Error>; - fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { loop { match self.neighbor_packet_worker.lock().poll_next_unpin(cx) { Poll::Ready(Some((to, packet))) => { @@ -444,6 +442,12 @@ impl> Future for NetworkBridge { } } + match self.gossip_engine.poll_unpin(cx) { + // The gossip engine future finished. We should do the same. + Poll::Ready(()) => return Poll::Ready(Ok(())), + Poll::Pending => {}, + } + Poll::Pending } } diff --git a/client/finality-grandpa/src/communication/periodic.rs b/client/finality-grandpa/src/communication/periodic.rs index 4635899690..f2e79e8f14 100644 --- a/client/finality-grandpa/src/communication/periodic.rs +++ b/client/finality-grandpa/src/communication/periodic.rs @@ -19,7 +19,7 @@ use futures_timer::Delay; use futures::{channel::mpsc, future::{FutureExt as _}, prelude::*, ready, stream::Stream}; use log::debug; -use std::{pin::Pin, task::{Context, Poll}, time::{Instant, Duration}}; +use std::{pin::Pin, task::{Context, Poll}, time::Duration}; use sc_network::PeerId; use sp_runtime::traits::{NumberFor, Block as BlockT}; @@ -28,10 +28,6 @@ use super::gossip::{NeighborPacket, GossipMessage}; // How often to rebroadcast, in cases where no new packets are created. const REBROADCAST_AFTER: Duration = Duration::from_secs(2 * 60); -fn rebroadcast_instant() -> Instant { - Instant::now() + REBROADCAST_AFTER -} - /// A sender used to send neighbor packets to a background job. #[derive(Clone)] pub(super) struct NeighborPacketSender( @@ -85,7 +81,7 @@ impl Stream for NeighborPacketWorker { match this.rx.poll_next_unpin(cx) { Poll::Ready(None) => return Poll::Ready(None), Poll::Ready(Some((to, packet))) => { - this.delay.reset(rebroadcast_instant()); + this.delay.reset(REBROADCAST_AFTER); this.last = Some((to.clone(), packet.clone())); return Poll::Ready(Some((to, GossipMessage::::from(packet.clone())))); @@ -98,7 +94,7 @@ impl Stream for NeighborPacketWorker { // Getting this far here implies that the timer fired. - this.delay.reset(rebroadcast_instant()); + this.delay.reset(REBROADCAST_AFTER); // Make sure the underlying task is scheduled for wake-up. // diff --git a/client/finality-grandpa/src/communication/tests.rs b/client/finality-grandpa/src/communication/tests.rs index e10d24a16a..040ee4c7bb 100644 --- a/client/finality-grandpa/src/communication/tests.rs +++ b/client/finality-grandpa/src/communication/tests.rs @@ -165,7 +165,7 @@ fn voter_set_state() -> SharedVoterSetState { } // needs to run in a tokio runtime. -pub(crate) fn make_test_network(executor: &impl futures::task::Spawn) -> ( +pub(crate) fn make_test_network() -> ( impl Future, TestNetwork, ) { @@ -187,7 +187,6 @@ pub(crate) fn make_test_network(executor: &impl futures::task::Spawn) -> ( net.clone(), config(), voter_set_state(), - executor, ); ( @@ -261,8 +260,7 @@ fn good_commit_leads_to_relay() { let id = sc_network::PeerId::random(); let global_topic = super::global_topic::(set_id); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let test = make_test_network(&threads_pool).0 + let test = make_test_network().0 .then(move |tester| { // register a peer. tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); @@ -281,6 +279,7 @@ fn good_commit_leads_to_relay() { } let commit_to_send = encoded_commit.clone(); + let network_bridge = tester.net_handle.clone(); // asking for global communication will cause the test network // to send us an event asking us for a stream. use it to @@ -325,7 +324,7 @@ fn good_commit_leads_to_relay() { // once the message is sent and commit is "handled" we should have // a repropagation event coming from the network. - future::join(send_message, handle_commit).then(move |(tester, ())| { + let fut = future::join(send_message, handle_commit).then(move |(tester, ())| { tester.filter_network_events(move |event| match event { Event::WriteNotification(_, data) => { data == encoded_commit @@ -333,7 +332,11 @@ fn good_commit_leads_to_relay() { _ => false, }) }) - .map(|_| ()) + .map(|_| ()); + + // Poll both the future sending and handling the commit, as well as the underlying + // NetworkBridge. Complete once the former completes. + future::select(fut, network_bridge) }); futures::executor::block_on(test); @@ -385,8 +388,7 @@ fn bad_commit_leads_to_report() { let id = sc_network::PeerId::random(); let global_topic = super::global_topic::(set_id); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let test = make_test_network(&threads_pool).0 + let test = make_test_network().0 .map(move |tester| { // register a peer. tester.gossip_validator.new_peer(&mut NoopContext, &id, sc_network::config::Roles::FULL); @@ -405,6 +407,7 @@ fn bad_commit_leads_to_report() { } let commit_to_send = encoded_commit.clone(); + let network_bridge = tester.net_handle.clone(); // asking for global communication will cause the test network // to send us an event asking us for a stream. use it to @@ -427,7 +430,7 @@ fn bad_commit_leads_to_report() { _ => false, }); - // when the commit comes in, we'll tell the callback it was good. + // when the commit comes in, we'll tell the callback it was bad. let handle_commit = commits_in.into_future() .map(|(item, _)| { match item.unwrap() { @@ -440,7 +443,7 @@ fn bad_commit_leads_to_report() { // once the message is sent and commit is "handled" we should have // a report event coming from the network. - future::join(send_message, handle_commit).then(move |(tester, ())| { + let fut = future::join(send_message, handle_commit).then(move |(tester, ())| { tester.filter_network_events(move |event| match event { Event::Report(who, cost_benefit) => { who == id && cost_benefit == super::cost::INVALID_COMMIT @@ -448,7 +451,11 @@ fn bad_commit_leads_to_report() { _ => false, }) }) - .map(|_| ()) + .map(|_| ()); + + // Poll both the future sending and handling the commit, as well as the underlying + // NetworkBridge. Complete once the former completes. + future::select(fut, network_bridge) }); futures::executor::block_on(test); @@ -458,8 +465,7 @@ fn bad_commit_leads_to_report() { fn peer_with_higher_view_leads_to_catch_up_request() { let id = sc_network::PeerId::random(); - let threads_pool = futures::executor::ThreadPool::new().unwrap(); - let (tester, mut net) = make_test_network(&threads_pool); + let (tester, mut net) = make_test_network(); let test = tester .map(move |tester| { // register a peer with authority role. diff --git a/client/finality-grandpa/src/lib.rs b/client/finality-grandpa/src/lib.rs index afca0ed48b..45a2400226 100644 --- a/client/finality-grandpa/src/lib.rs +++ b/client/finality-grandpa/src/lib.rs @@ -516,7 +516,7 @@ fn register_finality_tracker_inherent_data_provider( } /// Parameters used to run Grandpa. -pub struct GrandpaParams { +pub struct GrandpaParams { /// Configuration for the GRANDPA service. pub config: Config, /// A link to the block import worker. @@ -531,14 +531,12 @@ pub struct GrandpaParams { pub telemetry_on_connect: Option>, /// A voting rule used to potentially restrict target votes. pub voting_rule: VR, - /// How to spawn background tasks. - pub executor: Sp, } /// Run a GRANDPA voter as a task. Provide configuration and a link to a /// block import worker that has already been instantiated with `block_import`. -pub fn run_grandpa_voter( - grandpa_params: GrandpaParams, +pub fn run_grandpa_voter( + grandpa_params: GrandpaParams, ) -> sp_blockchain::Result + Unpin + Send + 'static> where Block::Hash: Ord, B: Backend + 'static, @@ -551,7 +549,6 @@ pub fn run_grandpa_voter( RA: Send + Sync + 'static, X: futures::Future + Clone + Send + Unpin + 'static, Client: AuxStore, - Sp: futures::task::Spawn + 'static, { let GrandpaParams { config, @@ -561,7 +558,6 @@ pub fn run_grandpa_voter( on_exit, telemetry_on_connect, voting_rule, - executor, } = grandpa_params; let LinkHalf { @@ -575,7 +571,6 @@ pub fn run_grandpa_voter( network, config.clone(), persistent_data.set_state.clone(), - &executor, ); register_finality_tracker_inherent_data_provider(client.clone(), &inherent_data_providers)?; @@ -863,8 +858,8 @@ where } #[deprecated(since = "1.1.0", note = "Please switch to run_grandpa_voter.")] -pub fn run_grandpa( - grandpa_params: GrandpaParams, +pub fn run_grandpa( + grandpa_params: GrandpaParams, ) -> sp_blockchain::Result + Send + 'static> where Block::Hash: Ord, B: Backend + 'static, @@ -877,7 +872,6 @@ pub fn run_grandpa( VR: VotingRule> + Clone + 'static, X: futures::Future + Clone + Send + Unpin + 'static, Client: AuxStore, - Sp: futures::task::Spawn + 'static, { run_grandpa_voter(grandpa_params) } diff --git a/client/finality-grandpa/src/observer.rs b/client/finality-grandpa/src/observer.rs index 7e6d1e7793..ffe71d573a 100644 --- a/client/finality-grandpa/src/observer.rs +++ b/client/finality-grandpa/src/observer.rs @@ -150,12 +150,11 @@ fn grandpa_observer( /// listening for and validating GRANDPA commits instead of following the full /// protocol. Provide configuration and a link to a block import worker that has /// already been instantiated with `block_import`. -pub fn run_grandpa_observer( +pub fn run_grandpa_observer( config: Config, link: LinkHalf, network: N, on_exit: impl futures::Future + Clone + Send + Unpin + 'static, - executor: Sp, ) -> sp_blockchain::Result + Unpin + Send + 'static> where B: Backend + 'static, E: CallExecutor + Send + Sync + 'static, @@ -163,7 +162,6 @@ pub fn run_grandpa_observer( SC: SelectChain + 'static, NumberFor: BlockNumberOps, RA: Send + Sync + 'static, - Sp: futures::task::Spawn + 'static, Client: AuxStore, { let LinkHalf { @@ -177,7 +175,6 @@ pub fn run_grandpa_observer( network, config.clone(), persistent_data.set_state.clone(), - &executor, ); let observer_work = ObserverWork::new( @@ -392,10 +389,8 @@ mod tests { /// network. #[test] fn observer_work_polls_underlying_network_bridge() { - let thread_pool = ThreadPool::new().unwrap(); - // Create a test network. - let (tester_fut, _network) = make_test_network(&thread_pool); + let (tester_fut, _network) = make_test_network(); let mut tester = executor::block_on(tester_fut); // Create an observer. diff --git a/client/finality-grandpa/src/tests.rs b/client/finality-grandpa/src/tests.rs index 0f4d9dadd0..cf340c6954 100644 --- a/client/finality-grandpa/src/tests.rs +++ b/client/finality-grandpa/src/tests.rs @@ -387,7 +387,6 @@ fn block_until_complete(future: impl Future + Unpin, net: &Arc( runtime: &mut current_thread::Runtime, - threads_pool: &futures::executor::ThreadPool, blocks: u64, net: Arc>, peers: &[Ed25519Keyring], @@ -454,7 +453,6 @@ fn run_to_completion_with( on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); @@ -473,12 +471,11 @@ fn run_to_completion_with( fn run_to_completion( runtime: &mut current_thread::Runtime, - threads_pool: &futures::executor::ThreadPool, blocks: u64, net: Arc>, peers: &[Ed25519Keyring] ) -> u64 { - run_to_completion_with(runtime, threads_pool, blocks, net, peers, |_| None) + run_to_completion_with(runtime, blocks, net, peers, |_| None) } fn add_scheduled_change(block: &mut Block, change: ScheduledChange) { @@ -499,17 +496,10 @@ fn add_forced_change( )); } -fn thread_pool() -> futures::executor::ThreadPool { - futures::executor::ThreadPool::builder().pool_size(2) - .create() - .expect("never fails") -} - #[test] fn finalize_3_voters_no_observers() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -523,7 +513,7 @@ fn finalize_3_voters_no_observers() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 20, net.clone(), peers); + run_to_completion(&mut runtime, 20, net.clone(), peers); // normally there's no justification for finalized blocks assert!( @@ -535,7 +525,6 @@ fn finalize_3_voters_no_observers() { #[test] fn finalize_3_voters_1_full_observer() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -595,7 +584,6 @@ fn finalize_3_voters_1_full_observer() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; voters.push(run_grandpa_voter(grandpa_params).expect("all in order with client and network")); @@ -641,7 +629,6 @@ fn transition_3_voters_twice_1_full_observer() { let net = Arc::new(Mutex::new(GrandpaTestNet::new(api, 8))); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); net.lock().peer(0).push_blocks(1, false); net.lock().block_until_sync(&mut runtime); @@ -760,7 +747,6 @@ fn transition_3_voters_twice_1_full_observer() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params).expect("all in order with client and network"); @@ -776,7 +762,6 @@ fn transition_3_voters_twice_1_full_observer() { #[test] fn justification_is_emitted_when_consensus_data_changes() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 3); @@ -785,7 +770,7 @@ fn justification_is_emitted_when_consensus_data_changes() { net.peer(0).push_authorities_change_block(new_authorities); net.block_until_sync(&mut runtime); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 1, net.clone(), peers); + run_to_completion(&mut runtime, 1, net.clone(), peers); // ... and check that there's justification for block#1 assert!(net.lock().peer(0).client().justification(&BlockId::Number(1)).unwrap().is_some(), @@ -795,7 +780,6 @@ fn justification_is_emitted_when_consensus_data_changes() { #[test] fn justification_is_generated_periodically() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(peers); @@ -804,7 +788,7 @@ fn justification_is_generated_periodically() { net.block_until_sync(&mut runtime); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 32, net.clone(), peers); + run_to_completion(&mut runtime, 32, net.clone(), peers); // when block#32 (justification_period) is finalized, justification // is required => generated @@ -835,7 +819,6 @@ fn consensus_changes_works() { #[test] fn sync_justifications_on_change_blocks() { let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers_b); @@ -867,7 +850,7 @@ fn sync_justifications_on_change_blocks() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 25, net.clone(), peers_a); + run_to_completion(&mut runtime, 25, net.clone(), peers_a); // the first 3 peers are grandpa voters and therefore have already finalized // block 21 and stored a justification @@ -890,7 +873,6 @@ fn sync_justifications_on_change_blocks() { fn finalizes_multiple_pending_changes_in_order() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers_a = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let peers_b = &[Ed25519Keyring::Dave, Ed25519Keyring::Eve, Ed25519Keyring::Ferdie]; @@ -944,14 +926,13 @@ fn finalizes_multiple_pending_changes_in_order() { } let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 30, net.clone(), all_peers); + run_to_completion(&mut runtime, 30, net.clone(), all_peers); } #[test] fn force_change_to_new_set() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // two of these guys are offline. let genesis_authorities = &[ Ed25519Keyring::Alice, @@ -1002,7 +983,7 @@ fn force_change_to_new_set() { // it will only finalize if the forced transition happens. // we add_blocks after the voters are spawned because otherwise // the link-halfs have the wrong AuthoritySet - run_to_completion(&mut runtime, &threads_pool, 25, net, peers_a); + run_to_completion(&mut runtime, 25, net, peers_a); } #[test] @@ -1132,7 +1113,6 @@ fn voter_persists_its_votes() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // we have two authorities but we'll only be running the voter for alice // we are going to be listening for the prevotes it casts @@ -1171,7 +1151,6 @@ fn voter_persists_its_votes() { net: Arc>, client: PeersClient, keystore: KeyStorePtr, - threads_pool: futures::executor::ThreadPool, } impl Future for ResettableVoter { @@ -1210,7 +1189,6 @@ fn voter_persists_its_votes() { on_exit: Exit, telemetry_on_connect: None, voting_rule: VotingRulesBuilder::default().build(), - executor: this.threads_pool.clone(), }; let voter = run_grandpa_voter(grandpa_params) @@ -1242,7 +1220,6 @@ fn voter_persists_its_votes() { net: net.clone(), client: client.clone(), keystore, - threads_pool: threads_pool.clone(), }.unit_error().compat()); } @@ -1278,7 +1255,6 @@ fn voter_persists_its_votes() { net.lock().peers[1].network_service().clone(), config.clone(), set_state, - &threads_pool, ); let (round_rx, round_tx) = network.round_communication( @@ -1289,6 +1265,11 @@ fn voter_persists_its_votes() { HasVoted::No, ); + runtime.spawn( + network.map_err(|e| panic!("network bridge should not error: {:?}", e)) + .compat(), + ); + let round_tx = Arc::new(Mutex::new(round_tx)); let exit_tx = Arc::new(Mutex::new(Some(exit_tx))); @@ -1389,7 +1370,6 @@ fn voter_persists_its_votes() { fn finalize_3_voters_1_light_observer() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let authorities = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob, Ed25519Keyring::Charlie]; let voters = make_ids(authorities); @@ -1406,10 +1386,12 @@ fn finalize_3_voters_1_light_observer() { let link = net.lock().peer(3).data.lock().take().expect("link initialized on startup; qed"); let finality_notifications = net.lock().peer(3).client().finality_notification_stream() - .take_while(|n| future::ready(n.header.number() < &20)) + .take_while(|n| { + future::ready(n.header.number() < &20) + }) .collect::>(); - run_to_completion_with(&mut runtime, &threads_pool, 20, net.clone(), authorities, |executor| { + run_to_completion_with(&mut runtime, 20, net.clone(), authorities, |executor| { executor.spawn( run_grandpa_observer( Config { @@ -1423,7 +1405,6 @@ fn finalize_3_voters_1_light_observer() { link, net.lock().peers[3].network_service().clone(), Exit, - threads_pool.clone(), ).unwrap().unit_error().compat() ).unwrap(); @@ -1435,7 +1416,6 @@ fn finalize_3_voters_1_light_observer() { fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { let _ = ::env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice]; let mut net = GrandpaTestNet::new(TestApi::new(make_ids(peers)), 1); @@ -1445,7 +1425,7 @@ fn finality_proof_is_fetched_by_light_client_when_consensus_data_changes() { // && instead fetches finality proof for block #1 net.peer(0).push_authorities_change_block(vec![sp_consensus_babe::AuthorityId::from_slice(&[42; 32])]); let net = Arc::new(Mutex::new(net)); - run_to_completion(&mut runtime, &threads_pool, 1, net.clone(), peers); + run_to_completion(&mut runtime, 1, net.clone(), peers); net.lock().block_until_sync(&mut runtime); // check that the block#1 is finalized on light client @@ -1467,7 +1447,6 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ let _ = ::env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); // two of these guys are offline. let genesis_authorities = if FORCE_CHANGE { @@ -1515,7 +1494,7 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ net.lock().block_until_sync(&mut runtime); // finalize block #11 on full clients - run_to_completion(&mut runtime, &threads_pool, 11, net.clone(), peers_a); + run_to_completion(&mut runtime, 11, net.clone(), peers_a); // request finalization by light client net.lock().add_light_peer(&GrandpaTestNet::default_config()); @@ -1532,7 +1511,6 @@ fn empty_finality_proof_is_returned_to_light_client_when_authority_set_is_differ fn voter_catches_up_to_latest_round_when_behind() { let _ = env_logger::try_init(); let mut runtime = current_thread::Runtime::new().unwrap(); - let threads_pool = thread_pool(); let peers = &[Ed25519Keyring::Alice, Ed25519Keyring::Bob]; let voters = make_ids(peers); @@ -1560,7 +1538,6 @@ fn voter_catches_up_to_latest_round_when_behind() { on_exit: Exit, telemetry_on_connect: None, voting_rule: (), - executor: threads_pool.clone(), }; Box::pin(run_grandpa_voter(grandpa_params).expect("all in order with client and network")) @@ -1652,8 +1629,6 @@ fn grandpa_environment_respects_voting_rules() { use finality_grandpa::Chain; use sc_network_test::TestClient; - let threads_pool = thread_pool(); - let peers = &[Ed25519Keyring::Alice]; let voters = make_ids(peers); @@ -1684,7 +1659,6 @@ fn grandpa_environment_respects_voting_rules() { network_service.clone(), config.clone(), set_state.clone(), - &threads_pool, ); Environment { diff --git a/client/informant/Cargo.toml b/client/informant/Cargo.toml new file mode 100644 index 0000000000..197f320889 --- /dev/null +++ b/client/informant/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "sc-informant" +version = "0.8.0" +authors = ["Parity Technologies "] +description = "Substrate informant." +edition = "2018" +license = "GPL-3.0" + +[dependencies] +ansi_term = "0.12.1" +futures = "0.3.1" +log = "0.4.8" +parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } +wasm-timer = "0.2" +sc-client-api = { version = "2.0.0", path = "../api" } +sc-network = { version = "0.8", path = "../network" } +sc-service = { version = "0.8", default-features = false, path = "../service" } +sp-blockchain = { version = "2.0.0", path = "../../primitives/blockchain" } +sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } diff --git a/client/cli/src/informant/display.rs b/client/informant/src/display.rs similarity index 76% rename from client/cli/src/informant/display.rs rename to client/informant/src/display.rs index 199635e7c7..53c9697868 100644 --- a/client/cli/src/informant/display.rs +++ b/client/informant/src/display.rs @@ -20,7 +20,9 @@ use log::info; use sc_network::SyncState; use sp_runtime::traits::{Block as BlockT, CheckedDiv, NumberFor, Zero, Saturating}; use sc_service::NetworkStatus; -use std::{convert::{TryFrom, TryInto}, fmt, time}; +use std::{convert::{TryFrom, TryInto}, fmt}; +use wasm_timer::Instant; +use crate::OutputFormat; /// State of the informant display system. /// @@ -40,15 +42,18 @@ pub struct InformantDisplay { /// `None` if `display` has never been called. last_number: Option>, /// The last time `display` or `new` has been called. - last_update: time::Instant, + last_update: Instant, + /// The format to print output in. + format: OutputFormat, } impl InformantDisplay { /// Builds a new informant display system. - pub fn new() -> InformantDisplay { + pub fn new(format: OutputFormat) -> InformantDisplay { InformantDisplay { last_number: None, - last_update: time::Instant::now(), + last_update: Instant::now(), + format, } } @@ -56,8 +61,10 @@ impl InformantDisplay { pub fn display(&mut self, info: &ClientInfo, net_status: NetworkStatus) { let best_number = info.chain.best_number; let best_hash = info.chain.best_hash; + let finalized_number = info.chain.finalized_number; + let num_connected_peers = net_status.num_connected_peers; let speed = speed::(best_number, self.last_number, self.last_update); - self.last_update = time::Instant::now(); + self.last_update = Instant::now(); self.last_number = Some(best_number); let (status, target) = match (net_status.sync_state, net_status.best_seen_block) { @@ -66,19 +73,35 @@ impl InformantDisplay { (SyncState::Downloading, Some(n)) => (format!("Syncing{}", speed), format!(", target=#{}", n)), }; - info!( - target: "substrate", - "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", - Colour::White.bold().paint(&status), - target, - Colour::White.bold().paint(format!("{}", net_status.num_connected_peers)), - Colour::White.paint(format!("{}", best_number)), - best_hash, - Colour::White.paint(format!("{}", info.chain.finalized_number)), - info.chain.finalized_hash, - TransferRateFormat(net_status.average_download_per_sec), - TransferRateFormat(net_status.average_upload_per_sec), - ); + if self.format == OutputFormat::Coloured { + info!( + target: "substrate", + "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + Colour::White.bold().paint(&status), + target, + Colour::White.bold().paint(format!("{}", num_connected_peers)), + Colour::White.paint(format!("{}", best_number)), + best_hash, + Colour::White.paint(format!("{}", finalized_number)), + info.chain.finalized_hash, + TransferRateFormat(net_status.average_download_per_sec), + TransferRateFormat(net_status.average_upload_per_sec), + ); + } else { + info!( + target: "substrate", + "{}{} ({} peers), best: #{} ({}), finalized #{} ({}), ⬇ {} ⬆ {}", + status, + target, + num_connected_peers, + best_number, + best_hash, + finalized_number, + info.chain.finalized_hash, + TransferRateFormat(net_status.average_download_per_sec), + TransferRateFormat(net_status.average_upload_per_sec), + ); + } } } @@ -87,7 +110,7 @@ impl InformantDisplay { fn speed( best_number: NumberFor, last_number: Option>, - last_update: time::Instant + last_update: Instant ) -> String { // Number of milliseconds elapsed since last time. let elapsed_ms = { diff --git a/client/cli/src/informant.rs b/client/informant/src/lib.rs similarity index 90% rename from client/cli/src/informant.rs rename to client/informant/src/lib.rs index 9e7c5044e0..699dcfdd74 100644 --- a/client/cli/src/informant.rs +++ b/client/informant/src/lib.rs @@ -25,12 +25,19 @@ use std::time::Duration; mod display; +/// The format to print telemetry output in. +#[derive(PartialEq)] +pub enum OutputFormat { + Coloured, + Plain, +} + /// Creates an informant in the form of a `Future` that must be polled regularly. -pub fn build(service: &impl AbstractService) -> impl futures::Future { +pub fn build(service: &impl AbstractService, format: OutputFormat) -> impl futures::Future { let client = service.client(); let pool = service.transaction_pool(); - let mut display = display::InformantDisplay::new(); + let mut display = display::InformantDisplay::new(format); let display_notifications = service .network_status(Duration::from_millis(5000)) @@ -41,6 +48,7 @@ pub fn build(service: &impl AbstractService) -> impl futures::Future. use crate::{Network, Validator}; -use crate::state_machine::{ConsensusGossip, TopicNotification}; +use crate::state_machine::{ConsensusGossip, TopicNotification, PERIODIC_MAINTENANCE_INTERVAL}; use sc_network::message::generic::ConsensusMessage; use sc_network::{Event, ReputationChange}; -use futures::{prelude::*, channel::mpsc, compat::Compat01As03, task::SpawnExt as _}; +use futures::{prelude::*, channel::mpsc, compat::Compat01As03}; +use futures01::stream::Stream as Stream01; use libp2p::PeerId; use parking_lot::Mutex; use sp_runtime::{traits::Block as BlockT, ConsensusEngineId}; -use std::{sync::Arc, time::Duration}; +use std::{pin::Pin, sync::Arc, task::{Context, Poll}}; /// Wraps around an implementation of the `Network` crate and provides gossiping capabilities on /// top of it. @@ -36,13 +37,17 @@ pub struct GossipEngine { struct GossipEngineInner { state_machine: ConsensusGossip, network: Box + Send>, + periodic_maintenance_interval: futures_timer::Delay, + network_event_stream: Compat01As03 + Send>>, + engine_id: ConsensusEngineId, } +impl Unpin for GossipEngineInner {} + impl GossipEngine { /// Create a new instance. pub fn new + Send + Clone + 'static>( mut network: N, - executor: &impl futures::task::Spawn, engine_id: ConsensusEngineId, validator: Arc>, ) -> Self where B: 'static { @@ -50,7 +55,7 @@ impl GossipEngine { // We grab the event stream before registering the notifications protocol, otherwise we // might miss events. - let event_stream = network.event_stream(); + let network_event_stream = network.event_stream(); network.register_notifications_protocol(engine_id); state_machine.register_validator(&mut network, engine_id, validator); @@ -58,6 +63,9 @@ impl GossipEngine { let inner = Arc::new(Mutex::new(GossipEngineInner { state_machine, network: Box::new(network), + periodic_maintenance_interval: futures_timer::Delay::new(PERIODIC_MAINTENANCE_INTERVAL), + network_event_stream: Compat01As03::new(network_event_stream), + engine_id, })); let gossip_engine = GossipEngine { @@ -65,72 +73,6 @@ impl GossipEngine { engine_id, }; - let res = executor.spawn({ - let inner = Arc::downgrade(&inner); - async move { - loop { - let _ = futures_timer::Delay::new(Duration::from_millis(1100)).await; - if let Some(inner) = inner.upgrade() { - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.tick(&mut *inner.network); - } else { - // We reach this branch if the `Arc` has no reference - // left. We can now let the task end. - break; - } - } - } - }); - - // Note: we consider the chances of an error to spawn a background task almost null. - if res.is_err() { - log::error!(target: "gossip", "Failed to spawn background task"); - } - - let res = executor.spawn(async move { - let mut stream = Compat01As03::new(event_stream); - while let Some(Ok(event)) = stream.next().await { - match event { - Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { - if msg_engine_id != engine_id { - continue; - } - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.new_peer(&mut *inner.network, remote, roles); - } - Event::NotificationStreamClosed { remote, engine_id: msg_engine_id } => { - if msg_engine_id != engine_id { - continue; - } - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.peer_disconnected(&mut *inner.network, remote); - }, - Event::NotificationsReceived { remote, messages } => { - let mut inner = inner.lock(); - let inner = &mut *inner; - inner.state_machine.on_incoming( - &mut *inner.network, - remote, - messages.into_iter() - .filter_map(|(engine, data)| if engine == engine_id { - Some(ConsensusMessage { engine_id: engine, data: data.to_vec() }) - } else { None }) - .collect() - ); - }, - Event::Dht(_) => {} - } - } - }); - - // Note: we consider the chances of an error to spawn a background task almost null. - if res.is_err() { - log::error!(target: "gossip", "Failed to spawn background task"); - } - gossip_engine } @@ -222,6 +164,59 @@ impl GossipEngine { } } +impl Future for GossipEngine { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll { + self.inner.lock().poll_unpin(cx) + } +} + +impl Future for GossipEngineInner { + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll { + let this = &mut *self; + + while let Poll::Ready(Some(Ok(event))) = this.network_event_stream.poll_next_unpin(cx) { + match event { + Event::NotificationStreamOpened { remote, engine_id: msg_engine_id, roles } => { + if msg_engine_id != this.engine_id { + continue; + } + this.state_machine.new_peer(&mut *this.network, remote, roles); + } + Event::NotificationStreamClosed { remote, engine_id: msg_engine_id } => { + if msg_engine_id != this.engine_id { + continue; + } + this.state_machine.peer_disconnected(&mut *this.network, remote); + }, + Event::NotificationsReceived { remote, messages } => { + let engine_id = this.engine_id.clone(); + this.state_machine.on_incoming( + &mut *this.network, + remote, + messages.into_iter() + .filter_map(|(engine, data)| if engine == engine_id { + Some(ConsensusMessage { engine_id: engine, data: data.to_vec() }) + } else { None }) + .collect() + ); + }, + Event::Dht(_) => {} + } + } + + while let Poll::Ready(()) = this.periodic_maintenance_interval.poll_unpin(cx) { + this.periodic_maintenance_interval.reset(PERIODIC_MAINTENANCE_INTERVAL); + this.state_machine.tick(&mut *this.network); + } + + Poll::Pending + } +} + impl Clone for GossipEngine { fn clone(&self) -> Self { GossipEngine { diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index d1931b1bd2..2acfdc3785 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -28,12 +28,15 @@ use sp_runtime::traits::{Block as BlockT, Hash, HashFor}; use sp_runtime::ConsensusEngineId; pub use sc_network::message::generic::{Message, ConsensusMessage}; use sc_network::config::Roles; +use wasm_timer::Instant; // FIXME: Add additional spam/DoS attack protection: https://github.com/paritytech/substrate/issues/1115 const KNOWN_MESSAGES_CACHE_SIZE: usize = 4096; const REBROADCAST_INTERVAL: time::Duration = time::Duration::from_secs(30); +pub(crate) const PERIODIC_MAINTENANCE_INTERVAL: time::Duration = time::Duration::from_millis(1100); + mod rep { use sc_network::ReputationChange as Rep; /// Reputation change when a peer sends us a gossip message that we didn't know about. @@ -165,7 +168,7 @@ pub struct ConsensusGossip { messages: Vec>, known_messages: LruCache, validators: HashMap>>, - next_broadcast: time::Instant, + next_broadcast: Instant, } impl ConsensusGossip { @@ -177,7 +180,7 @@ impl ConsensusGossip { messages: Default::default(), known_messages: LruCache::new(KNOWN_MESSAGES_CACHE_SIZE), validators: Default::default(), - next_broadcast: time::Instant::now() + REBROADCAST_INTERVAL, + next_broadcast: Instant::now() + REBROADCAST_INTERVAL, } } @@ -260,9 +263,9 @@ impl ConsensusGossip { /// Perform periodic maintenance pub fn tick(&mut self, network: &mut dyn Network) { self.collect_garbage(); - if time::Instant::now() >= self.next_broadcast { + if Instant::now() >= self.next_broadcast { self.rebroadcast(network); - self.next_broadcast = time::Instant::now() + REBROADCAST_INTERVAL; + self.next_broadcast = Instant::now() + REBROADCAST_INTERVAL; } } diff --git a/client/network/Cargo.toml b/client/network/Cargo.toml index e506a61c75..f6ff3bcb73 100644 --- a/client/network/Cargo.toml +++ b/client/network/Cargo.toml @@ -6,6 +6,9 @@ license = "GPL-3.0" authors = ["Parity Technologies "] edition = "2018" +[build-dependencies] +prost-build = "0.6.1" + [dependencies] bitflags = "1.2.0" bytes = "0.5.0" @@ -17,13 +20,16 @@ fnv = "1.0.6" fork-tree = { version = "2.0.0", path = "../../utils/fork-tree" } futures = "0.3.1" futures_codec = "0.3.3" -futures-timer = "0.4.0" +futures-timer = "3.0.1" +wasm-timer = "0.2" libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } linked-hash-map = "0.5.2" linked_hash_set = "0.1.3" log = "0.4.8" lru = "0.4.0" +nohash-hasher = "0.1.3" parking_lot = "0.10.0" +prost = "0.6.1" rand = "0.7.2" rustc-hex = "2.0.1" sc-block-builder = { version = "0.8", path = "../block-builder" } @@ -44,16 +50,22 @@ sp-keyring = { version = "2.0.0", optional = true, path = "../../primitives/keyr sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } substrate-test-client = { version = "2.0.0", optional = true, path = "../../test-utils/client" } substrate-test-runtime-client = { version = "2.0.0", optional = true, path = "../../test-utils/runtime/client" } +thiserror = "1" unsigned-varint = { version = "0.3.0", features = ["codec"] } void = "1.0.2" zeroize = "1.0.0" +yamux = "0.4.2" [dev-dependencies] +async-std = "1.5" +assert_matches = "1.3" env_logger = "0.7.0" quickcheck = "0.9.0" rand = "0.7.2" sp-keyring = { version = "2.0.0", path = "../../primitives/keyring" } sp-test-primitives = { version = "2.0.0", path = "../../primitives/test-primitives" } +substrate-test-runtime = { version = "2.0.0", path = "../../test-utils/runtime" } +substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/runtime/client" } tempfile = "3.1.0" [features] diff --git a/client/network/build.rs b/client/network/build.rs new file mode 100644 index 0000000000..0fd1f12866 --- /dev/null +++ b/client/network/build.rs @@ -0,0 +1,8 @@ +const PROTOS: &[&str] = &[ + "src/protocol/schema/api.v1.proto", + "src/protocol/schema/light.v1.proto" +]; + +fn main() { + prost_build::compile_protos(PROTOS, &["src/protocol"]).unwrap(); +} diff --git a/client/network/src/behaviour.rs b/client/network/src/behaviour.rs index 8b903cec35..63cbea4da5 100644 --- a/client/network/src/behaviour.rs +++ b/client/network/src/behaviour.rs @@ -19,7 +19,7 @@ use crate::{ Event, protocol::event::DhtEvent }; use crate::{ExHashT, specialization::NetworkSpecialization}; -use crate::protocol::{CustomMessageOutcome, Protocol}; +use crate::protocol::{self, light_client_handler, CustomMessageOutcome, Protocol}; use libp2p::NetworkBehaviour; use libp2p::core::{Multiaddr, PeerId, PublicKey}; use libp2p::kad::record; @@ -42,7 +42,10 @@ pub struct Behaviour, H: ExHashT> { debug_info: debug_info::DebugInfoBehaviour>, /// Discovers nodes of the network. discovery: DiscoveryBehaviour>, - + /// Block request handling. + block_requests: protocol::BlockRequests, B>, + /// Light client request handling. + light_client_handler: protocol::LightClientHandler, B>, /// Queue of events to produce for the outside. #[behaviour(ignore)] events: Vec>, @@ -65,6 +68,9 @@ impl, H: ExHashT> Behaviour { known_addresses: Vec<(PeerId, Multiaddr)>, enable_mdns: bool, allow_private_ipv4: bool, + discovery_only_if_under_num: u64, + block_requests: protocol::BlockRequests, B>, + light_client_handler: protocol::LightClientHandler, B>, ) -> Self { Behaviour { substrate, @@ -73,9 +79,12 @@ impl, H: ExHashT> Behaviour { local_public_key, known_addresses, enable_mdns, - allow_private_ipv4 + allow_private_ipv4, + discovery_only_if_under_num, ).await, - events: Vec::new(), + block_requests, + light_client_handler, + events: Vec::new() } } @@ -117,6 +126,12 @@ impl, H: ExHashT> Behaviour { pub fn put_value(&mut self, key: record::Key, value: Vec) { self.discovery.put_value(key, value); } + + /// Issue a light client request. + #[allow(unused)] + pub fn light_client_request(&mut self, r: light_client_handler::Request) -> Result<(), light_client_handler::Error> { + self.light_client_handler.request(r) + } } impl, H: ExHashT> NetworkBehaviourEventProcess for diff --git a/client/network/src/config.rs b/client/network/src/config.rs index 6cf2587fe4..87c77fee9f 100644 --- a/client/network/src/config.rs +++ b/client/network/src/config.rs @@ -292,6 +292,7 @@ impl Default for NetworkConfiguration { enable_mdns: false, allow_private_ipv4: true, wasm_external_transport: None, + use_yamux_flow_control: false, }, max_parallel_downloads: 5, } @@ -348,6 +349,8 @@ pub enum TransportConfig { /// This parameter exists whatever the target platform is, but it is expected to be set to /// `Some` only when compiling for WASM. wasm_external_transport: Option, + /// Use flow control for yamux streams if set to true. + use_yamux_flow_control: bool, }, /// Only allow connections within the same process. diff --git a/client/network/src/debug_info.rs b/client/network/src/debug_info.rs index b06e275d1d..a3d63333fa 100644 --- a/client/network/src/debug_info.rs +++ b/client/network/src/debug_info.rs @@ -28,7 +28,8 @@ use std::error; use std::collections::hash_map::Entry; use std::pin::Pin; use std::task::{Context, Poll}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; use crate::utils::interval; /// Time after we disconnect from a node before we purge its information from the cache. diff --git a/client/network/src/discovery.rs b/client/network/src/discovery.rs index de49913b26..2da69e1894 100644 --- a/client/network/src/discovery.rs +++ b/client/network/src/discovery.rs @@ -87,6 +87,8 @@ pub struct DiscoveryBehaviour { /// If false, `addresses_of_peer` won't return any private IPv4 address, except for the ones /// stored in `user_defined`. allow_private_ipv4: bool, + /// Number of active connections over which we interrupt the discovery process. + discovery_only_if_under_num: u64, } impl DiscoveryBehaviour { @@ -98,6 +100,7 @@ impl DiscoveryBehaviour { user_defined: Vec<(PeerId, Multiaddr)>, enable_mdns: bool, allow_private_ipv4: bool, + discovery_only_if_under_num: u64, ) -> Self { if enable_mdns { #[cfg(target_os = "unknown")] @@ -120,6 +123,7 @@ impl DiscoveryBehaviour { local_peer_id: local_public_key.into_peer_id(), num_connections: 0, allow_private_ipv4, + discovery_only_if_under_num, #[cfg(not(target_os = "unknown"))] mdns: if enable_mdns { match Mdns::new() { @@ -331,11 +335,19 @@ where // Poll the stream that fires when we need to start a random Kademlia query. while let Poll::Ready(_) = self.next_kad_random_query.poll_unpin(cx) { - let random_peer_id = PeerId::random(); - debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ - {:?}", random_peer_id); + if self.num_connections < self.discovery_only_if_under_num { + let random_peer_id = PeerId::random(); + debug!(target: "sub-libp2p", "Libp2p <= Starting random Kademlia request for \ + {:?}", random_peer_id); - self.kademlia.get_closest_peers(random_peer_id); + self.kademlia.get_closest_peers(random_peer_id); + } else { + debug!( + target: "sub-libp2p", + "Kademlia paused due to high number of connections ({})", + self.num_connections + ); + } // Schedule the next random query with exponentially increasing delay, // capped at 60 seconds. @@ -435,6 +447,10 @@ where NetworkBehaviourAction::GenerateEvent(event) => { match event { MdnsEvent::Discovered(list) => { + if self.num_connections >= self.discovery_only_if_under_num { + continue; + } + self.discoveries.extend(list.into_iter().map(|(peer_id, _)| peer_id)); if let Some(peer_id) = self.discoveries.pop_front() { let ev = DiscoveryOut::Discovered(peer_id); @@ -502,7 +518,7 @@ mod tests { let user_defined = user_defined.clone(); let keypair_public = keypair.public(); async move { - DiscoveryBehaviour::new(keypair_public, user_defined, false, true).await + DiscoveryBehaviour::new(keypair_public, user_defined, false, true, 50).await } }); let mut swarm = Swarm::new(transport, behaviour, keypair.public().into_peer_id()); diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 5e8df2831b..8e5f9d6e7b 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -52,16 +52,32 @@ use crate::chain::{Client, FinalityProofProvider}; use sc_client_api::{FetchChecker, ChangesProof, StorageProof}; use crate::error; use util::LruHashSet; +use wasm_timer::Instant; + +// Include sources generated from protobuf definitions. +pub mod api { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/api.v1.rs")); + pub mod light { + include!(concat!(env!("OUT_DIR"), "/api.v1.light.rs")); + } + } +} mod legacy_proto; mod util; +pub mod block_requests; pub mod message; pub mod event; +pub mod light_client_handler; pub mod light_dispatch; pub mod specialization; pub mod sync; +pub use block_requests::BlockRequests; +pub use light_client_handler::LightClientHandler; + const REQUEST_TIMEOUT_SEC: u64 = 40; /// Interval at which we perform time based maintenance const TICK_TIMEOUT: time::Duration = time::Duration::from_millis(1100); @@ -158,7 +174,7 @@ struct PacketStats { /// A peer that we are connected to /// and from whom we have not yet received a Status message. struct HandshakingPeer { - timestamp: time::Instant, + timestamp: Instant, } /// Peer information @@ -166,9 +182,9 @@ struct HandshakingPeer { struct Peer { info: PeerInfo, /// Current block request, if any. - block_request: Option<(time::Instant, message::BlockRequest)>, + block_request: Option<(Instant, message::BlockRequest)>, /// Requests we are no longer insterested in. - obsolete_requests: HashMap, + obsolete_requests: HashMap, /// Holds a set of transactions known to this peer. known_extrinsics: LruHashSet, /// Holds a set of blocks known to this peer. @@ -701,7 +717,7 @@ impl, H: ExHashT> Protocol { /// Called when a new peer is connected pub fn on_peer_connected(&mut self, who: PeerId) { trace!(target: "sync", "Connecting {}", who); - self.handshaking_peers.insert(who.clone(), HandshakingPeer { timestamp: time::Instant::now() }); + self.handshaking_peers.insert(who.clone(), HandshakingPeer { timestamp: Instant::now() }); self.send_status(who); } @@ -890,7 +906,7 @@ impl, H: ExHashT> Protocol { } fn maintain_peers(&mut self) { - let tick = time::Instant::now(); + let tick = Instant::now(); let mut aborting = Vec::new(); { for (who, peer) in self.context_data.peers.iter() { @@ -1833,7 +1849,7 @@ fn send_request( trace!(target: "sync", "Request {} for {} is now obsolete.", request.id, who); peer.obsolete_requests.insert(request.id, timestamp); } - peer.block_request = Some((time::Instant::now(), r.clone())); + peer.block_request = Some((Instant::now(), r.clone())); } } send_message::(behaviour, stats, who, message) diff --git a/client/network/src/protocol/block_requests.rs b/client/network/src/protocol/block_requests.rs new file mode 100644 index 0000000000..f8a905c288 --- /dev/null +++ b/client/network/src/protocol/block_requests.rs @@ -0,0 +1,354 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! `NetworkBehaviour` implementation which handles incoming block requests. +//! +//! Every request is coming in on a separate connection substream which gets +//! closed after we have sent the response back. Incoming requests are encoded +//! as protocol buffers (cf. `api.v1.proto`). + +#![allow(unused)] + +use bytes::Bytes; +use codec::{Encode, Decode}; +use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, message::BlockAttributes} +}; +use futures::{future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use libp2p::{ + core::{ + ConnectedPoint, + Multiaddr, + PeerId, + upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + upgrade::{DeniedUpgrade, read_one, write_one} + }, + swarm::{NetworkBehaviour, NetworkBehaviourAction, OneShotHandler, PollParameters, SubstreamProtocol} +}; +use prost::Message; +use sp_runtime::{generic::BlockId, traits::{Block, Header, One, Zero}}; +use std::{ + cmp::min, + io, + iter, + sync::Arc, + time::Duration, + task::{Context, Poll} +}; +use void::{Void, unreachable}; + +// Type alias for convenience. +pub type Error = Box; + +/// Configuration options for `BlockRequests`. +#[derive(Debug, Clone)] +pub struct Config { + max_block_data_response: u32, + max_request_len: usize, + inactivity_timeout: Duration, + protocol: Bytes, +} + +impl Config { + /// Create a fresh configuration with the following options: + /// + /// - max. block data in response = 128 + /// - max. request size = 1 MiB + /// - inactivity timeout = 15s + pub fn new(id: &ProtocolId) -> Self { + let mut c = Config { + max_block_data_response: 128, + max_request_len: 1024 * 1024, + inactivity_timeout: Duration::from_secs(15), + protocol: Bytes::new(), + }; + c.set_protocol(id); + c + } + + /// Limit the max. number of block data in a response. + pub fn set_max_block_data_response(&mut self, v: u32) -> &mut Self { + self.max_block_data_response = v; + self + } + + /// Limit the max. length of incoming block request bytes. + pub fn set_max_request_len(&mut self, v: usize) -> &mut Self { + self.max_request_len = v; + self + } + + /// Limit the max. duration the substream may remain inactive before closing it. + pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { + self.inactivity_timeout = v; + self + } + + /// Set protocol to use for upgrade negotiation. + pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { + let mut v = Vec::new(); + v.extend_from_slice(b"/"); + v.extend_from_slice(id.as_bytes()); + v.extend_from_slice(b"/sync/1"); + self.protocol = v.into(); + self + } +} + +/// The block request handling behaviour. +pub struct BlockRequests { + /// This behaviour's configuration. + config: Config, + /// Blockchain client. + chain: Arc>, + /// Futures sending back the block request response. + outgoing: FuturesUnordered>, + /// Type witness term. + _marker: std::marker::PhantomData +} + +impl BlockRequests +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, + B: Block, +{ + pub fn new(cfg: Config, chain: Arc>) -> Self { + BlockRequests { + config: cfg, + chain, + outgoing: FuturesUnordered::new(), + _marker: std::marker::PhantomData + } + } + + /// Callback, invoked when a new block request has been received from remote. + fn on_block_request + ( &mut self + , peer: &PeerId + , request: &api::v1::BlockRequest + ) -> Result + { + log::trace!("block request {} from peer {}: from block {:?} to block {:?}, max blocks {:?}", + request.id, + peer, + request.from_block, + request.to_block, + request.max_blocks); + + let from_block_id = + match request.from_block { + Some(api::v1::block_request::FromBlock::Hash(ref h)) => { + let h = Decode::decode(&mut h.as_ref())?; + BlockId::::Hash(h) + } + Some(api::v1::block_request::FromBlock::Number(ref n)) => { + let n = Decode::decode(&mut n.as_ref())?; + BlockId::::Number(n) + } + None => { + let msg = "missing `BlockRequest::from_block` field"; + return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + } + }; + + let max_blocks = + if request.max_blocks == 0 { + self.config.max_block_data_response + } else { + min(request.max_blocks, self.config.max_block_data_response) + }; + + let direction = + if request.direction == api::v1::Direction::Ascending as i32 { + api::v1::Direction::Ascending + } else if request.direction == api::v1::Direction::Descending as i32 { + api::v1::Direction::Descending + } else { + let msg = format!("invalid `BlockRequest::direction` value: {}", request.direction); + return Err(io::Error::new(io::ErrorKind::Other, msg).into()) + }; + + let attributes = BlockAttributes::decode(&mut request.fields.to_be_bytes().as_ref())?; + let get_header = attributes.contains(BlockAttributes::HEADER); + let get_body = attributes.contains(BlockAttributes::BODY); + let get_justification = attributes.contains(BlockAttributes::JUSTIFICATION); + + let mut blocks = Vec::new(); + let mut block_id = from_block_id; + while let Some(header) = self.chain.header(&block_id).unwrap_or(None) { + if blocks.len() >= max_blocks as usize { + break + } + + let number = header.number().clone(); + let hash = header.hash(); + let parent_hash = header.parent_hash().clone(); + + let block_data = api::v1::BlockData { + hash: hash.encode(), + header: if get_header { + header.encode() + } else { + Vec::new() + }, + body: if get_body { + self.chain.body(&BlockId::Hash(hash))? + .unwrap_or(Vec::new()) + .iter_mut() + .map(|extrinsic| extrinsic.encode()) + .collect() + } else { + Vec::new() + }, + receipt: Vec::new(), + message_queue: Vec::new(), + justification: if get_justification { + self.chain.justification(&BlockId::Hash(hash))?.unwrap_or(Vec::new()) + } else { + Vec::new() + } + }; + + blocks.push(block_data); + + match direction { + api::v1::Direction::Ascending => { + block_id = BlockId::Number(number + One::one()) + } + api::v1::Direction::Descending => { + if number.is_zero() { + break + } + block_id = BlockId::Hash(parent_hash) + } + } + } + + Ok(api::v1::BlockResponse { id: request.id, blocks }) + } +} + +impl NetworkBehaviour for BlockRequests +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, + B: Block +{ + type ProtocolsHandler = OneShotHandler>>; + type OutEvent = Void; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + let p = Protocol { + max_request_len: self.config.max_request_len, + protocol: self.config.protocol.clone(), + }; + OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + } + + fn addresses_of_peer(&mut self, _: &PeerId) -> Vec { + Vec::new() + } + + fn inject_connected(&mut self, _peer: PeerId, _info: ConnectedPoint) { + } + + fn inject_disconnected(&mut self, _peer: &PeerId, _info: ConnectedPoint) { + } + + fn inject_node_event(&mut self, peer: PeerId, Request(request, mut stream): Request>) { + match self.on_block_request(&peer, &request) { + Ok(res) => { + log::trace!("enqueueing block response {} for peer {} with {} blocks", res.id, peer, res.blocks.len()); + let mut data = Vec::with_capacity(res.encoded_len()); + if let Err(e) = res.encode(&mut data) { + log::debug!("error encoding block response {} for peer {}: {}", res.id, peer, e) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!("error writing block response: {}", e) + } + }; + self.outgoing.push(future.boxed()) + } + } + Err(e) => log::debug!("error handling block request {} from peer {}: {}", request.id, peer, e) + } + } + + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) -> Poll> { + while let Poll::Ready(Some(_)) = self.outgoing.poll_next_unpin(cx) {} + Poll::Pending + } +} + +/// The incoming block request. +/// +/// Holds the protobuf value and the connection substream which made the +/// request and over which to send the response. +#[derive(Debug)] +pub struct Request(api::v1::BlockRequest, T); + +impl From for Request { + fn from(v: Void) -> Self { + unreachable(v) + } +} + +/// Substream upgrade protocol. +/// +/// We attempt to parse an incoming protobuf encoded request (cf. `Request`) +/// which will be handled by the `BlockRequests` behaviour, i.e. the request +/// will become visible via `inject_node_event` which then dispatches to the +/// relevant callback to process the message and prepare a response. +#[derive(Debug, Clone)] +pub struct Protocol { + /// The max. request length in bytes. + max_request_len: usize, + /// The protocol to use during upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for Protocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl InboundUpgrade for Protocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Request; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + let len = self.max_request_len; + let vec = read_one(&mut s, len).await?; + match api::v1::BlockRequest::decode(&vec[..]) { + Ok(r) => Ok(Request(r, s)), + Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) + } + }; + future.boxed() + } +} + diff --git a/client/network/src/protocol/legacy_proto/behaviour.rs b/client/network/src/protocol/legacy_proto/behaviour.rs index b4047f320a..1eb6b157dd 100644 --- a/client/network/src/protocol/legacy_proto/behaviour.rs +++ b/client/network/src/protocol/legacy_proto/behaviour.rs @@ -26,7 +26,8 @@ use log::{debug, error, trace, warn}; use rand::distributions::{Distribution as _, Uniform}; use smallvec::SmallVec; use std::{borrow::Cow, collections::hash_map::Entry, cmp, error, marker::PhantomData, mem, pin::Pin}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; use std::task::{Context, Poll}; /// Network behaviour that handles opening substreams for custom protocols with other nodes. @@ -387,7 +388,7 @@ impl LegacyProto { debug!(target: "sub-libp2p", "PSM => Connect({:?}): Will start to connect at \ until {:?}", occ_entry.key(), until); *occ_entry.into_mut() = PeerState::PendingRequest { - timer: futures_timer::Delay::new_at(until.clone()), + timer: futures_timer::Delay::new(until.clone() - Instant::now()), timer_deadline: until.clone(), }; }, @@ -406,7 +407,7 @@ impl LegacyProto { *occ_entry.into_mut() = PeerState::DisabledPendingEnable { connected_point: connected_point.clone(), open, - timer: futures_timer::Delay::new_at(banned.clone()), + timer: futures_timer::Delay::new(banned.clone() - Instant::now()), timer_deadline: banned.clone(), }; }, diff --git a/client/network/src/protocol/legacy_proto/handler.rs b/client/network/src/protocol/legacy_proto/handler.rs index fc3b64b968..66fb0dca13 100644 --- a/client/network/src/protocol/legacy_proto/handler.rs +++ b/client/network/src/protocol/legacy_proto/handler.rs @@ -348,13 +348,12 @@ where ProtocolState::Init { substreams, mut init_deadline } => { match Pin::new(&mut init_deadline).poll(cx) { - Poll::Ready(Ok(())) => { + Poll::Ready(()) => { init_deadline = Delay::new(Duration::from_secs(60)); error!(target: "sub-libp2p", "Handler initialization process is too long \ with {:?}", self.remote_peer_id) }, Poll::Pending => {} - Poll::Ready(Err(_)) => error!(target: "sub-libp2p", "Tokio timer has errored") } self.state = ProtocolState::Init { substreams, init_deadline }; @@ -363,7 +362,7 @@ where ProtocolState::Opening { mut deadline } => { match Pin::new(&mut deadline).poll(cx) { - Poll::Ready(Ok(())) => { + Poll::Ready(()) => { deadline = Delay::new(Duration::from_secs(60)); let event = CustomProtoHandlerOut::ProtocolError { is_severe: true, @@ -376,12 +375,6 @@ where self.state = ProtocolState::Opening { deadline }; None }, - Poll::Ready(Err(_)) => { - error!(target: "sub-libp2p", "Tokio timer has errored"); - deadline = Delay::new(Duration::from_secs(60)); - self.state = ProtocolState::Opening { deadline }; - None - }, } } diff --git a/client/network/src/protocol/legacy_proto/tests.rs b/client/network/src/protocol/legacy_proto/tests.rs index 18e32f1d01..4f523fa31d 100644 --- a/client/network/src/protocol/legacy_proto/tests.rs +++ b/client/network/src/protocol/legacy_proto/tests.rs @@ -409,7 +409,7 @@ fn reconnect_after_disconnect() { _ => panic!() } - if let Poll::Ready(Ok(_)) = delay.poll_unpin(cx) { + if let Poll::Ready(()) = delay.poll_unpin(cx) { Poll::Ready(Ok(())) } else { Poll::Pending diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs new file mode 100644 index 0000000000..f5be23c0d4 --- /dev/null +++ b/client/network/src/protocol/light_client_handler.rs @@ -0,0 +1,1793 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. +// +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! [`NetworkBehaviour`] implementation which handles light client requests. +//! +//! Every request is coming in on a separate connection substream which gets +//! closed after we have sent the response back. Requests and responses are +//! encoded as protocol buffers (cf. `api.v1.proto`). +//! +//! For every outgoing request we likewise open a separate substream. + +#![allow(unused)] + +use bytes::Bytes; +use codec::{self, Encode, Decode}; +use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, light_dispatch::TIMEOUT_REPUTATION_CHANGE} +}; +use futures::{channel::oneshot, future::BoxFuture, prelude::*, stream::FuturesUnordered}; +use libp2p::{ + core::{ + ConnectedPoint, + Multiaddr, + PeerId, + upgrade::{InboundUpgrade, ReadOneError, UpgradeInfo, Negotiated}, + upgrade::{OutboundUpgrade, read_one, write_one} + }, + swarm::{NetworkBehaviour, NetworkBehaviourAction, OneShotHandler, PollParameters, SubstreamProtocol} +}; +use nohash_hasher::IntMap; +use prost::Message; +use rustc_hex::ToHex; +use sc_client::light::fetcher; +use sc_client_api::StorageProof; +use sc_peerset::ReputationChange; +use sp_core::storage::{ChildInfo, StorageKey}; +use sp_blockchain::{Error as ClientError}; +use sp_runtime::traits::{Block, Header, NumberFor, Zero}; +use std::{ + collections::{BTreeMap, VecDeque, HashMap}, + iter, + io, + sync::Arc, + time::{Duration, Instant}, + task::{Context, Poll} +}; +use void::Void; + +/// Configuration options for `LightClientHandler` behaviour. +#[derive(Debug, Clone)] +pub struct Config { + max_data_size: usize, + max_pending_requests: usize, + inactivity_timeout: Duration, + request_timeout: Duration, + protocol: Bytes, +} + +impl Config { + /// Create a fresh configuration with the following options: + /// + /// - max. data size = 1 MiB + /// - max. pending requests = 128 + /// - inactivity timeout = 15s + /// - request timeout = 15s + pub fn new(id: &ProtocolId) -> Self { + let mut c = Config { + max_data_size: 1024 * 1024, + max_pending_requests: 128, + inactivity_timeout: Duration::from_secs(15), + request_timeout: Duration::from_secs(15), + protocol: Bytes::new(), + }; + c.set_protocol(id); + c + } + + /// Limit the max. length of incoming request bytes. + pub fn set_max_data_size(&mut self, v: usize) -> &mut Self { + self.max_data_size = v; + self + } + + /// Limit the max. number of pending requests. + pub fn set_max_pending_requests(&mut self, v: usize) -> &mut Self { + self.max_pending_requests = v; + self + } + + /// Limit the max. duration the connection may remain inactive before closing it. + pub fn set_inactivity_timeout(&mut self, v: Duration) -> &mut Self { + self.inactivity_timeout = v; + self + } + + /// Limit the max. request duration. + pub fn set_request_timeout(&mut self, v: Duration) -> &mut Self { + self.request_timeout = v; + self + } + + /// Set protocol to use for upgrade negotiation. + pub fn set_protocol(&mut self, id: &ProtocolId) -> &mut Self { + let mut v = Vec::new(); + v.extend_from_slice(b"/"); + v.extend_from_slice(id.as_bytes()); + v.extend_from_slice(b"/light/1"); + self.protocol = v.into(); + self + } +} + +/// Possible errors while handling light clients. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// There are currently too many pending request. + #[error("too many pending requests")] + TooManyRequests, + /// The response type does not correspond to the issued request. + #[error("unexpected response")] + UnexpectedResponse, + /// A bad request has been received. + #[error("bad request: {0}")] + BadRequest(&'static str), + /// The chain client errored. + #[error("client error: {0}")] + Client(#[from] ClientError), + /// Encoding or decoding of some data failed. + #[error("codec error: {0}")] + Codec(#[from] codec::Error), +} + +/// The possible light client requests we support. +/// +/// The associated `oneshot::Sender` will be used to convey the result of +/// their request back to them (cf. `Reply`). +// +// This is modeled after light_dispatch.rs's `RequestData` which is not +// used because we currently only support a subset of those. +#[derive(Debug)] +pub enum Request { + Header { + request: fetcher::RemoteHeaderRequest, + sender: oneshot::Sender> + }, + Read { + request: fetcher::RemoteReadRequest, + sender: oneshot::Sender, Option>>, ClientError>> + }, + ReadChild { + request: fetcher::RemoteReadChildRequest, + sender: oneshot::Sender, Option>>, ClientError>> + }, + Call { + request: fetcher::RemoteCallRequest, + sender: oneshot::Sender, ClientError>> + }, + Changes { + request: fetcher::RemoteChangesRequest, + sender: oneshot::Sender, u32)>, ClientError>> + } +} + +/// The data to send back to the light client over the oneshot channel. +// +// It is unified here in order to be able to return it as a function +// result instead of delivering it to the client as a side effect of +// response processing. +#[derive(Debug)] +enum Reply { + VecU8(Vec), + VecNumberU32(Vec<(::Number, u32)>), + MapVecU8OptVecU8(HashMap, Option>>), + Header(B::Header) +} + +/// Augments a light client request with metadata. +#[derive(Debug)] +struct RequestWrapper { + /// Time when this value was created. + timestamp: Instant, + /// Remaining retries. + retries: usize, + /// The actual request. + request: Request, + /// Peer information, e.g. `PeerId`. + peer: P +} + +/// Information we have about some peer. +#[derive(Debug)] +struct PeerInfo { + address: Multiaddr, + best_block: Option>, + status: PeerStatus, +} + +/// A peer is either idle or busy processing a request from us. +#[derive(Debug, Clone, PartialEq, Eq)] +enum PeerStatus { + /// The peer is available. + Idle, + /// We wait for the peer to return us a response for the given request ID. + BusyWith(u64), +} + +/// The light client handler behaviour. +pub struct LightClientHandler { + /// This behaviour's configuration. + config: Config, + /// Blockchain client. + chain: Arc>, + /// Verifies that received responses are correct. + checker: Arc>, + /// Peer information (addresses, their best block, etc.) + peers: HashMap>, + /// Futures sending back response to remote clients. + responses: FuturesUnordered>, + /// Pending (local) requests. + pending_requests: VecDeque>, + /// Requests on their way to remote peers. + outstanding: IntMap>, + /// (Local) Request ID counter + next_request_id: u64, + /// Handle to use for reporting misbehaviour of peers. + peerset: sc_peerset::PeersetHandle, + /// Type witness term. + _marker: std::marker::PhantomData +} + +impl LightClientHandler +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, + B: Block, +{ + /// Construct a new light client handler. + pub fn new + ( cfg: Config + , chain: Arc> + , checker: Arc> + , peerset: sc_peerset::PeersetHandle + ) -> Self + { + LightClientHandler { + config: cfg, + chain, + checker, + peers: HashMap::new(), + responses: FuturesUnordered::new(), + pending_requests: VecDeque::new(), + outstanding: IntMap::default(), + next_request_id: 1, + peerset, + _marker: std::marker::PhantomData + } + } + + /// We rely on external information about peers best blocks as we lack the + /// means to determine it ourselves. + pub fn update_best_block(&mut self, peer: &PeerId, num: NumberFor) { + if let Some(info) = self.peers.get_mut(peer) { + info.best_block = Some(num) + } + } + + /// Issue a new light client request. + pub fn request(&mut self, req: Request) -> Result<(), Error> { + if self.pending_requests.len() >= self.config.max_pending_requests { + return Err(Error::TooManyRequests) + } + let rw = RequestWrapper { + timestamp: Instant::now(), + retries: retries(&req), + request: req, + peer: (), // we do not know the peer yet + }; + self.pending_requests.push_back(rw); + Ok(()) + } + + fn next_request_id(&mut self) -> u64 { + let id = self.next_request_id; + self.next_request_id += 1; + id + } + + // Iterate over peers known to possess a certain block. + fn idle_peers_with_block(&mut self, num: NumberFor) -> impl Iterator + '_ { + self.peers.iter() + .filter(move |(_, info)| { + info.status == PeerStatus::Idle && info.best_block >= Some(num) + }) + .map(|(peer, _)| peer.clone()) + } + + // Iterate over peers without a known block. + fn idle_peers_with_unknown_block(&mut self) -> impl Iterator + '_ { + self.peers.iter() + .filter(|(_, info)| { + info.status == PeerStatus::Idle && info.best_block.is_none() + }) + .map(|(peer, _)| peer.clone()) + } + + /// Remove the given peer. + /// + /// If we have a request to this peer in flight, we move it back to + /// the pending requests queue. + fn remove_peer(&mut self, peer: &PeerId) { + if let Some(id) = self.outstanding.iter().find(|(_, rw)| &rw.peer == peer).map(|(k, _)| *k) { + let rw = self.outstanding.remove(&id).expect("key belongs to entry in this map"); + let rw = RequestWrapper { + timestamp: rw.timestamp, + retries: rw.retries, + request: rw.request, + peer: (), // need to find another peer + }; + self.pending_requests.push_back(rw); + } + self.peers.remove(peer); + } + + /// Process a local request's response from remote. + /// + /// If successful, this will give us the actual, checked data we should be + /// sending back to the client, otherwise an error. + fn on_response + ( &mut self + , peer: &PeerId + , request: &Request + , response: api::v1::light::Response + ) -> Result, Error> + { + log::trace!("response {} from {}", response.id, peer); + use api::v1::light::response::Response; + match response.response { + Some(Response::RemoteCallResponse(response)) => + if let Request::Call { request , .. } = request { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_execution_proof(request, proof)?; + Ok(Reply::VecU8(reply)) + } else { + Err(Error::UnexpectedResponse) + } + Some(Response::RemoteReadResponse(response)) => + match request { + Request::Read { request, .. } => { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_read_proof(&request, proof)?; + Ok(Reply::MapVecU8OptVecU8(reply)) + } + Request::ReadChild { request, .. } => { + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_read_child_proof(&request, proof)?; + Ok(Reply::MapVecU8OptVecU8(reply)) + } + _ => Err(Error::UnexpectedResponse) + } + Some(Response::RemoteChangesResponse(response)) => + if let Request::Changes { request, .. } = request { + let max_block = Decode::decode(&mut response.max.as_ref())?; + let roots_proof = Decode::decode(&mut response.roots_proof.as_ref())?; + let roots = { + let mut r = BTreeMap::new(); + for pair in response.roots { + let k = Decode::decode(&mut pair.fst.as_ref())?; + let v = Decode::decode(&mut pair.snd.as_ref())?; + r.insert(k, v); + } + r + }; + let reply = self.checker.check_changes_proof(&request, fetcher::ChangesProof { + max_block, + proof: response.proof, + roots, + roots_proof, + })?; + Ok(Reply::VecNumberU32(reply)) + } else { + Err(Error::UnexpectedResponse) + } + Some(Response::RemoteHeaderResponse(response)) => + if let Request::Header { request, .. } = request { + let header = + if response.header.is_empty() { + None + } else { + Some(Decode::decode(&mut response.header.as_ref())?) + }; + let proof = Decode::decode(&mut response.proof.as_ref())?; + let reply = self.checker.check_header_proof(&request, header, proof)?; + Ok(Reply::Header(reply)) + } else { + Err(Error::UnexpectedResponse) + } + None => Err(Error::UnexpectedResponse) + } + } + + fn on_remote_call_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteCallRequest + ) -> Result + { + log::trace!("remote call request {} from {} ({} at {:?})", + request_id, + peer, + request.method, + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = match self.chain.execution_proof(&block, &request.method, &request.data) { + Ok((_, proof)) => proof, + Err(e) => { + log::trace!("remote call request {} from {} ({} at {:?}) failed with: {}", + request_id, + peer, + request.method, + request.block, + e); + StorageProof::empty() + } + }; + + let response = { + let r = api::v1::light::RemoteCallResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteCallResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_read_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteReadRequest + ) -> Result + { + if request.keys.is_empty() { + log::debug!("invalid remote read request sent by {}", peer); + return Err(Error::BadRequest("remote read request without keys")) + } + + log::trace!("remote read request {} from {} ({} at {:?})", + request_id, + peer, + fmt_keys(request.keys.first(), request.keys.last()), + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = match self.chain.read_proof(&block, &request.keys) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote read request {} from {} ({} at {:?}) failed with: {}", + request_id, + peer, + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + error); + StorageProof::empty() + } + }; + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteReadResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_read_child_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteReadChildRequest + ) -> Result + { + if request.keys.is_empty() { + log::debug!("invalid remote child read request sent by {}", peer); + return Err(Error::BadRequest("remove read child request without keys")) + } + + log::trace!("remote read child request {} from {} ({} {} at {:?})", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let proof = + if let Some(info) = ChildInfo::resolve_child_info(request.child_type, &request.child_info[..]) { + match self.chain.read_child_proof(&block, &request.storage_key, info, &request.keys) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + error); + StorageProof::empty() + } + } + } else { + log::trace!("remote read child request {} from {} ({} {} at {:?}) failed with: {}", + request_id, + peer, + request.storage_key.to_hex::(), + fmt_keys(request.keys.first(), request.keys.last()), + request.block, + "invalid child info and type" + ); + StorageProof::empty() + }; + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: proof.encode() }; + api::v1::light::response::Response::RemoteReadResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_header_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteHeaderRequest + ) -> Result + { + log::trace!("remote header proof request {} from {} ({:?})", request_id, peer, request.block); + + let block = Decode::decode(&mut request.block.as_ref())?; + + let (header, proof) = match self.chain.header_proof(block) { + Ok((header, proof)) => (header.encode(), proof), + Err(error) => { + log::trace!("remote header proof request {} from {} ({:?}) failed with: {}", + request_id, + peer, + request.block, + error); + (Default::default(), StorageProof::empty()) + } + }; + + let response = { + let r = api::v1::light::RemoteHeaderResponse { header, proof: proof.encode() }; + api::v1::light::response::Response::RemoteHeaderResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } + + fn on_remote_changes_request + ( &mut self + , peer: &PeerId + , request_id: u64 + , request: &api::v1::light::RemoteChangesRequest + ) -> Result + { + log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?})", + request_id, + peer, + if !request.storage_key.is_empty() { + format!("{} : {}", request.storage_key.to_hex::(), request.key.to_hex::()) + } else { + request.key.to_hex::() + }, + request.first, + request.last); + + let first = Decode::decode(&mut request.first.as_ref())?; + let last = Decode::decode(&mut request.last.as_ref())?; + let min = Decode::decode(&mut request.min.as_ref())?; + let max = Decode::decode(&mut request.max.as_ref())?; + let key = StorageKey(request.key.clone()); + let storage_key = + if request.storage_key.is_empty() { + None + } else { + Some(StorageKey(request.storage_key.clone())) + }; + + let proof = match self.chain.key_changes_proof(first, last, min, max, storage_key.as_ref(), &key) { + Ok(proof) => proof, + Err(error) => { + log::trace!("remote changes proof request {} from {} for key {} ({:?}..{:?}) failed with: {}", + request_id, + peer, + if let Some(sk) = storage_key { + format!("{} : {}", sk.0.to_hex::(), key.0.to_hex::()) + } else { + key.0.to_hex::() + }, + request.first, + request.last, + error); + + fetcher::ChangesProof:: { + max_block: Zero::zero(), + proof: Vec::new(), + roots: BTreeMap::new(), + roots_proof: StorageProof::empty(), + } + } + }; + + let response = { + let r = api::v1::light::RemoteChangesResponse { + max: proof.max_block.encode(), + proof: proof.proof, + roots: proof.roots.into_iter() + .map(|(k, v)| api::v1::light::Pair { fst: k.encode(), snd: v.encode() }) + .collect(), + roots_proof: proof.roots_proof.encode(), + }; + api::v1::light::response::Response::RemoteChangesResponse(r) + }; + + Ok(api::v1::light::Response { id: request_id, response: Some(response) }) + } +} + +impl NetworkBehaviour for LightClientHandler +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static, + B: Block +{ + type ProtocolsHandler = OneShotHandler>>; + type OutEvent = Void; + + fn new_handler(&mut self) -> Self::ProtocolsHandler { + let p = InboundProtocol { + max_data_size: self.config.max_data_size, + protocol: self.config.protocol.clone(), + }; + OneShotHandler::new(SubstreamProtocol::new(p), self.config.inactivity_timeout) + } + + fn addresses_of_peer(&mut self, peer: &PeerId) -> Vec { + self.peers.get(peer) + .map(|info| vec![info.address.clone()]) + .unwrap_or_default() + } + + fn inject_connected(&mut self, peer: PeerId, info: ConnectedPoint) { + let peer_address = match info { + ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr, + ConnectedPoint::Dialer { address } => address + }; + + log::trace!("peer {} connected with address {}", peer, peer_address); + + let info = PeerInfo { + address: peer_address, + best_block: None, + status: PeerStatus::Idle, + }; + + self.peers.insert(peer, info); + } + + fn inject_disconnected(&mut self, peer: &PeerId, _: ConnectedPoint) { + log::trace!("peer {} disconnected", peer); + self.remove_peer(peer) + } + + fn inject_node_event(&mut self, peer: PeerId, event: Event>) { + match event { + // An incoming request from remote has been received. + Event::Request(request, mut stream) => { + log::trace!("incoming request {} from {}", peer, request.id); + let result = match &request.request { + Some(api::v1::light::request::Request::RemoteCallRequest(r)) => + self.on_remote_call_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteReadRequest(r)) => + self.on_remote_read_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteHeaderRequest(r)) => + self.on_remote_header_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteReadChildRequest(r)) => + self.on_remote_read_child_request(&peer, request.id, r), + Some(api::v1::light::request::Request::RemoteChangesRequest(r)) => + self.on_remote_changes_request(&peer, request.id, r), + None => { + log::debug!("ignoring request {} without request data from peer {}", request.id, peer); + return + } + }; + match result { + Ok(response) => { + log::trace!("enqueueing response {} for peer {}", response.id, peer); + let mut data = Vec::new(); + if let Err(e) = response.encode(&mut data) { + log::debug!("error encoding response {} for peer {}: {}", response.id, peer, e) + } else { + let future = async move { + if let Err(e) = write_one(&mut stream, data).await { + log::debug!("error writing response: {}", e) + } + }; + self.responses.push(future.boxed()) + } + } + Err(Error::BadRequest(_)) => { + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new(-(1 << 12), "bad request")) + } + Err(e) => log::debug!("error handling request {} from peer {}: {}", request.id, peer, e) + } + } + // A response to one of our own requests has been received. + Event::Response(response) => { + let id = response.id; + if let Some(request) = self.outstanding.remove(&id) { + // We first just check if the response originates from the expected peer. + if request.peer != peer { + log::debug!("was expecting response {} from {} instead of {}", id, request.peer, peer); + self.outstanding.insert(id, request); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); + return + } + + if let Some(info) = self.peers.get_mut(&peer) { + if info.status != PeerStatus::BusyWith(id) { + // If we get here, something is wrong with our internal handling of peer + // status information. At any time, a single peer processes at most one + // request from us and its status should contain the request ID we are + // expecting a response for. If a peer would send us a response with a + // random ID, we should not have an entry for it with this peer ID in + // our `outstanding` map, so a malicious peer should not be able to get + // us here. It is our own fault and must be fixed! + panic!("unexpected peer status {:?} for {}", info.status, peer); + } + + info.status = PeerStatus::Idle; // Make peer available again. + + match self.on_response(&peer, &request.request, response) { + Ok(reply) => send_reply(Ok(reply), request.request), + Err(Error::UnexpectedResponse) => { + log::debug!("unexpected response {} from peer {}", id, peer); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("unexpected response from peer")); + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries, + request: request.request, + peer: (), + }; + self.pending_requests.push_back(rw); + } + Err(other) => { + log::debug!("error handling response {} from peer {}: {}", id, peer, other); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("invalid response from peer")); + if request.retries > 0 { + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries - 1, + request: request.request, + peer: (), + }; + self.pending_requests.push_back(rw) + } else { + send_reply(Err(ClientError::RemoteFetchFailed), request.request) + } + } + } + } else { + // If we get here, something is wrong with our internal handling of peers. + // We apparently have an entry in our `outstanding` map and the peer is the one we + // expected. So, if we can not find an entry for it in our peer information table, + // then these two collections are out of sync which must not happen and is a clear + // programmer error that must be fixed! + panic!("missing peer information for {}; response {}", peer, id); + } + } else { + log::debug!("unexpected response {} from peer {}", id, peer); + self.remove_peer(&peer); + self.peerset.report_peer(peer, ReputationChange::new_fatal("response from unexpected peer")); + } + } + } + } + + fn poll(&mut self, cx: &mut Context, _: &mut impl PollParameters) -> Poll> { + // Process response sending futures. + while let Poll::Ready(Some(_)) = self.responses.poll_next_unpin(cx) {} + + // If we have a pending request to send, try to find an available peer and send it. + let now = Instant::now(); + while let Some(mut request) = self.pending_requests.pop_front() { + if now > request.timestamp + self.config.request_timeout { + if request.retries == 0 { + send_reply(Err(ClientError::RemoteFetchFailed), request.request); + continue + } + request.timestamp = Instant::now(); + request.retries -= 1 + } + let number = required_block(&request.request); + let available_peer = { + let p = self.idle_peers_with_block(number).next(); + if p.is_none() { + self.idle_peers_with_unknown_block().next() + } else { + p + } + }; + if let Some(peer) = available_peer { + let id = self.next_request_id(); + let rq = serialise_request(id, &request.request); + let mut buf = Vec::with_capacity(rq.encoded_len()); + if let Err(e) = rq.encode(&mut buf) { + log::debug!("failed to serialise request {}: {}", id, e); + send_reply(Err(ClientError::RemoteFetchFailed), request.request) + } else { + log::trace!("sending request {} to peer {}", id, peer); + let protocol = OutboundProtocol { + request: buf, + max_data_size: self.config.max_data_size, + protocol: self.config.protocol.clone(), + }; + self.peers.get_mut(&peer).map(|info| info.status = PeerStatus::BusyWith(id)); + let rw = RequestWrapper { + timestamp: request.timestamp, + retries: request.retries, + request: request.request, + peer: peer.clone(), + }; + self.outstanding.insert(id, rw); + return Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id: peer, event: protocol }) + } + } else { + self.pending_requests.push_front(request); + log::debug!("no peer available to send request to"); + break + } + } + + // Look for ongoing requests that have timed out. + let mut expired = Vec::new(); + for (id, rw) in &self.outstanding { + if now > rw.timestamp + self.config.request_timeout { + log::debug!("request {} timed out", id); + expired.push(*id) + } + } + for id in expired { + if let Some(rw) = self.outstanding.remove(&id) { + self.remove_peer(&rw.peer); + self.peerset.report_peer(rw.peer.clone(), + ReputationChange::new(TIMEOUT_REPUTATION_CHANGE, "light request timeout")); + if rw.retries == 0 { + send_reply(Err(ClientError::RemoteFetchFailed), rw.request); + continue + } + let rw = RequestWrapper { + timestamp: Instant::now(), + retries: rw.retries - 1, + request: rw.request, + peer: (), + }; + self.pending_requests.push_back(rw) + } + } + + Poll::Pending + } +} + +fn required_block(request: &Request) -> NumberFor { + match request { + Request::Header { request, .. } => request.block, + Request::Read { request, .. } => *request.header.number(), + Request::ReadChild { request, .. } => *request.header.number(), + Request::Call { request, .. } => *request.header.number(), + Request::Changes { request, .. } => request.max_block.0, + } +} + +fn retries(request: &Request) -> usize { + let rc = match request { + Request::Header { request, .. } => request.retry_count, + Request::Read { request, .. } => request.retry_count, + Request::ReadChild { request, .. } => request.retry_count, + Request::Call { request, .. } => request.retry_count, + Request::Changes { request, .. } => request.retry_count, + }; + rc.unwrap_or(0) +} + +fn serialise_request(id: u64, request: &Request) -> api::v1::light::Request { + let request = match request { + Request::Header { request, .. } => { + let r = api::v1::light::RemoteHeaderRequest { block: request.block.encode() }; + api::v1::light::request::Request::RemoteHeaderRequest(r) + } + Request::Read { request, .. } => { + let r = api::v1::light::RemoteReadRequest { + block: request.block.encode(), + keys: request.keys.clone(), + }; + api::v1::light::request::Request::RemoteReadRequest(r) + } + Request::ReadChild { request, .. } => { + let r = api::v1::light::RemoteReadChildRequest { + block: request.block.encode(), + storage_key: request.storage_key.clone(), + child_type: request.child_type.clone(), + child_info: request.child_info.clone(), + keys: request.keys.clone(), + }; + api::v1::light::request::Request::RemoteReadChildRequest(r) + } + Request::Call { request, .. } => { + let r = api::v1::light::RemoteCallRequest { + block: request.block.encode(), + method: request.method.clone(), + data: request.call_data.clone(), + }; + api::v1::light::request::Request::RemoteCallRequest(r) + } + Request::Changes { request, .. } => { + let r = api::v1::light::RemoteChangesRequest { + first: request.first_block.1.encode(), + last: request.last_block.1.encode(), + min: request.tries_roots.1.encode(), + max: request.max_block.1.encode(), + storage_key: request.storage_key.clone().unwrap_or_default(), + key: request.key.clone(), + }; + api::v1::light::request::Request::RemoteChangesRequest(r) + } + }; + + api::v1::light::Request { id, request: Some(request) } +} + +fn send_reply(result: Result, ClientError>, request: Request) { + fn send(item: T, sender: oneshot::Sender) { + let _ = sender.send(item); // It is okay if the other end already hung up. + } + match request { + Request::Header { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::Header(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for header request: {:?}, {:?}", reply, request), + } + Request::Read { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for read request: {:?}, {:?}", reply, request), + } + Request::ReadChild { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::MapVecU8OptVecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for read child request: {:?}, {:?}", reply, request), + } + Request::Call { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::VecU8(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for call request: {:?}, {:?}", reply, request), + } + Request::Changes { request, sender } => match result { + Err(e) => send(Err(e), sender), + Ok(Reply::VecNumberU32(x)) => send(Ok(x), sender), + reply => log::error!("invalid reply for changes request: {:?}, {:?}", reply, request), + } + } +} + +/// Output type of inbound and outbound substream upgrades. +#[derive(Debug)] +pub enum Event { + /// Incoming request from remote and substream to use for the response. + Request(api::v1::light::Request, T), + /// Incoming response from remote. + Response(api::v1::light::Response), +} + +/// Substream upgrade protocol. +/// +/// Reads incoming requests from remote. +#[derive(Debug, Clone)] +pub struct InboundProtocol { + /// The max. request length in bytes. + max_data_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for InboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl InboundUpgrade for InboundProtocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_inbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + let vec = read_one(&mut s, self.max_data_size).await?; + match api::v1::light::Request::decode(&vec[..]) { + Ok(r) => Ok(Event::Request(r, s)), + Err(e) => Err(ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e))) + } + }; + future.boxed() + } +} + +/// Substream upgrade protocol. +/// +/// Sends a request to remote and awaits the response. +#[derive(Debug, Clone)] +pub struct OutboundProtocol { + /// The serialised protobuf request. + request: Vec, + /// The max. request length in bytes. + max_data_size: usize, + /// The protocol to use for upgrade negotiation. + protocol: Bytes, +} + +impl UpgradeInfo for OutboundProtocol { + type Info = Bytes; + type InfoIter = iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + iter::once(self.protocol.clone()) + } +} + +impl OutboundUpgrade for OutboundProtocol +where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static +{ + type Output = Event; + type Error = ReadOneError; + type Future = BoxFuture<'static, Result>; + + fn upgrade_outbound(self, mut s: T, _: Self::Info) -> Self::Future { + let future = async move { + write_one(&mut s, &self.request).await?; + let vec = read_one(&mut s, self.max_data_size).await?; + api::v1::light::Response::decode(&vec[..]) + .map(Event::Response) + .map_err(|e| { + ReadOneError::Io(io::Error::new(io::ErrorKind::Other, e)) + }) + }; + future.boxed() + } +} + +fn fmt_keys(first: Option<&Vec>, last: Option<&Vec>) -> String { + if let (Some(first), Some(last)) = (first, last) { + if first == last { + first.to_hex::() + } else { + format!("{}..{}", first.to_hex::(), last.to_hex::()) + } + } else { + String::from("n/a") + } +} + +#[cfg(test)] +mod tests { + use async_std::task; + use assert_matches::assert_matches; + use codec::Encode; + use crate::{ + chain::Client, + config::ProtocolId, + protocol::{api, light_dispatch::tests::{DummyFetchChecker, dummy_header}} + }; + use futures::{channel::oneshot, prelude::*}; + use libp2p::{ + PeerId, + Multiaddr, + core::{ + ConnectedPoint, + identity, + muxing::{StreamMuxerBox, SubstreamRef}, + transport::{Transport, boxed::Boxed, memory::MemoryTransport}, + upgrade + }, + noise::{self, Keypair, X25519, NoiseConfig}, + swarm::{NetworkBehaviour, NetworkBehaviourAction, PollParameters}, + yamux + }; + use sc_client_api::StorageProof; + use sc_client::light::fetcher; + use sp_blockchain::{Error as ClientError}; + use sp_core::storage::ChildInfo; + use std::{ + collections::HashSet, + io, + iter::{self, FromIterator}, + pin::Pin, + sync::Arc, + task::{Context, Poll} + }; + use sp_runtime::{generic::Header, traits::BlakeTwo256}; + use super::{Event, LightClientHandler, Request, OutboundProtocol, PeerStatus}; + use void::Void; + + const CHILD_INFO: ChildInfo<'static> = ChildInfo::new_default(b"foobarbaz"); + + type Block = sp_runtime::generic::Block, substrate_test_runtime::Extrinsic>; + type Handler = LightClientHandler>, Block>; + type Swarm = libp2p::swarm::Swarm, Handler>; + + fn empty_proof() -> Vec { + StorageProof::empty().encode() + } + + fn make_swarm(ok: bool, ps: sc_peerset::PeersetHandle, cf: super::Config) -> Swarm { + let client = Arc::new(substrate_test_runtime_client::new()); + let checker = Arc::new(DummyFetchChecker::new(ok)); + let id_key = identity::Keypair::generate_ed25519(); + let dh_key = Keypair::::new().into_authentic(&id_key).unwrap(); + let local_peer = id_key.public().into_peer_id(); + let transport = MemoryTransport::default() + .upgrade(upgrade::Version::V1) + .authenticate(NoiseConfig::xx(dh_key).into_authenticated()) + .multiplex(yamux::Config::default()) + .map(|(peer, muxer), _| (peer, StreamMuxerBox::new(muxer))) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e)) + .boxed(); + Swarm::new(transport, LightClientHandler::new(cf, client, checker, ps), local_peer) + } + + fn make_config() -> super::Config { + super::Config::new(&ProtocolId::from(&b"foo"[..])) + } + + struct EmptyPollParams(PeerId); + + impl PollParameters for EmptyPollParams { + type SupportedProtocolsIter = iter::Empty>; + type ListenedAddressesIter = iter::Empty; + type ExternalAddressesIter = iter::Empty; + + fn supported_protocols(&self) -> Self::SupportedProtocolsIter { + iter::empty() + } + + fn listened_addresses(&self) -> Self::ListenedAddressesIter { + iter::empty() + } + + fn external_addresses(&self) -> Self::ExternalAddressesIter { + iter::empty() + } + + fn local_peer_id(&self) -> &PeerId { + &self.0 + } + } + + fn peerset() -> (sc_peerset::Peerset, sc_peerset::PeersetHandle) { + let cfg = sc_peerset::PeersetConfig { + in_peers: 128, + out_peers: 128, + bootnodes: Vec::new(), + reserved_only: false, + reserved_nodes: Vec::new(), + }; + sc_peerset::Peerset::from_config(cfg) + } + + fn make_behaviour + ( ok: bool + , ps: sc_peerset::PeersetHandle + , cf: super::Config + ) -> LightClientHandler>, Block> + { + let client = Arc::new(substrate_test_runtime_client::new()); + let checker = Arc::new(DummyFetchChecker::new(ok)); + LightClientHandler::new(cf, client, checker, ps) + } + + fn empty_dialer() -> ConnectedPoint { + ConnectedPoint::Dialer { address: Multiaddr::empty() } + } + + fn poll(mut b: &mut LightClientHandler) -> Poll> + where + T: AsyncRead + AsyncWrite + Unpin + Send + 'static + { + let mut p = EmptyPollParams(PeerId::random()); + match future::poll_fn(|cx| Pin::new(&mut b).poll(cx, &mut p)).now_or_never() { + Some(a) => Poll::Ready(a), + None => Poll::Pending + } + } + + #[test] + fn disconnects_from_peer_if_told() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + behaviour.inject_disconnected(&peer, empty_dialer()); + assert_eq!(0, behaviour.peers.len()) + } + + #[test] + fn disconnects_from_peer_if_request_times_out() { + let peer0 = PeerId::random(); + let peer1 = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer0.clone(), empty_dialer()); + behaviour.inject_connected(peer1.clone(), empty_dialer()); + + // We now know about two peers. + assert_eq!(HashSet::from_iter(&[peer0.clone(), peer1.clone()]), behaviour.peers.keys().collect::>()); + + // No requests have been made yet. + assert!(behaviour.pending_requests.is_empty()); + assert!(behaviour.outstanding.is_empty()); + + // Issue our first request! + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + assert_eq!(1, behaviour.pending_requests.len()); + + // The behaviour should now attempt to send the request. + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { peer_id, .. }) => { + assert!(peer_id == peer0 || peer_id == peer1) + }); + + // And we should have one busy peer. + assert!({ + let (idle, busy): (Vec<_>, Vec<_>) = + behaviour.peers.iter().partition(|(_, info)| info.status == PeerStatus::Idle); + + idle.len() == 1 && busy.len() == 1 + && (idle[0].0 == &peer0 || busy[0].0 == &peer0) + && (idle[0].0 == &peer1 || busy[0].0 == &peer1) + }); + + // No more pending requests, but one should be outstanding. + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + // We now set back the timestamp of the outstanding request to make it expire. + let request = behaviour.outstanding.values_mut().next().unwrap(); + request.timestamp -= make_config().request_timeout; + + // Make progress, but do not expect some action. + assert_matches!(poll(&mut behaviour), Poll::Pending); + + // The request should have timed out by now and the corresponding peer be removed. + assert_eq!(1, behaviour.peers.len()); + // Since we asked for one retry, the request should be back in the pending queue. + assert_eq!(1, behaviour.pending_requests.len()); + // No other request should be ongoing. + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_response_with_wrong_id() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + // Construct response with bogus ID + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 2365789, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + // Make sure our bogus ID is really not used. + assert!(!behaviour.outstanding.keys().any(|id| id == &response.id)); + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_incorrect_response() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(false, pset.1, make_config()); + // ^--- Making sure the response data check fails. + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + let request_id = *behaviour.outstanding.keys().next().unwrap(); + + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_unexpected_response() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + + // Some unsolicited response + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 2347895932, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + + assert!(behaviour.peers.is_empty()); + poll(&mut behaviour); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn disconnects_from_peer_on_wrong_response_type() { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(1), + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + poll(&mut behaviour); // Make progress + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + let request_id = *behaviour.outstanding.keys().next().unwrap(); + + let response = { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; // Not a RemoteCallResponse! + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + }; + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + assert!(behaviour.peers.is_empty()); + + poll(&mut behaviour); // More progress + + // The request should be back in the pending queue + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + } + + #[test] + fn receives_remote_failure_after_retry_count_failures() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let peer3 = PeerId::random(); + let peer4 = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(false, pset.1, make_config()); + // ^--- Making sure the response data check fails. + + behaviour.inject_connected(peer1.clone(), empty_dialer()); + behaviour.inject_connected(peer2.clone(), empty_dialer()); + behaviour.inject_connected(peer3.clone(), empty_dialer()); + behaviour.inject_connected(peer4.clone(), empty_dialer()); + assert_eq!(4, behaviour.peers.len()); + + let mut chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: Some(3), // Attempt up to three retries. + }; + behaviour.request(Request::Call { request, sender: chan.0 }).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + + for _ in 0 .. 3 { + // Construct an invalid response + let request_id = *behaviour.outstanding.keys().next().unwrap(); + let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)) + } + }; + behaviour.inject_node_event(responding_peer, Event::Response(response.clone())); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_matches!(chan.1.try_recv(), Ok(None)) + } + // Final invalid response + let request_id = *behaviour.outstanding.keys().next().unwrap(); + let responding_peer = behaviour.outstanding.values().next().unwrap().peer.clone(); + let response = { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: request_id, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + }; + behaviour.inject_node_event(responding_peer, Event::Response(response)); + assert_matches!(poll(&mut behaviour), Poll::Pending); + assert_matches!(chan.1.try_recv(), Ok(Some(Err(ClientError::RemoteFetchFailed)))) + } + + fn issue_request(request: Request) { + let peer = PeerId::random(); + let pset = peerset(); + let mut behaviour = make_behaviour(true, pset.1, make_config()); + + behaviour.inject_connected(peer.clone(), empty_dialer()); + assert_eq!(1, behaviour.peers.len()); + + let response = match request { + Request::Header{..} => { + let r = api::v1::light::RemoteHeaderResponse { + header: dummy_header().encode(), + proof: empty_proof() + }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteHeaderResponse(r)), + } + } + Request::Read{..} => { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + } + Request::ReadChild{..} => { + let r = api::v1::light::RemoteReadResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteReadResponse(r)), + } + } + Request::Call{..} => { + let r = api::v1::light::RemoteCallResponse { proof: empty_proof() }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteCallResponse(r)), + } + } + Request::Changes{..} => { + let r = api::v1::light::RemoteChangesResponse { + max: iter::repeat(1).take(32).collect(), + proof: Vec::new(), + roots: Vec::new(), + roots_proof: empty_proof() + }; + api::v1::light::Response { + id: 1, + response: Some(api::v1::light::response::Response::RemoteChangesResponse(r)), + } + } + }; + + behaviour.request(request).unwrap(); + + assert_eq!(1, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()); + assert_matches!(poll(&mut behaviour), Poll::Ready(NetworkBehaviourAction::SendEvent { .. })); + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(1, behaviour.outstanding.len()); + assert_eq!(1, *behaviour.outstanding.keys().next().unwrap()); + + behaviour.inject_node_event(peer.clone(), Event::Response(response)); + + poll(&mut behaviour); + + assert_eq!(0, behaviour.pending_requests.len()); + assert_eq!(0, behaviour.outstanding.len()) + } + + #[test] + fn receives_remote_call_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }; + issue_request(Request::Call { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_read_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteReadRequest { + header: dummy_header(), + block: Default::default(), + keys: vec![b":key".to_vec()], + retry_count: None, + }; + issue_request(Request::Read { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_read_child_response() { + let info = CHILD_INFO.info(); + let mut chan = oneshot::channel(); + let request = fetcher::RemoteReadChildRequest { + header: dummy_header(), + block: Default::default(), + storage_key: b":child_storage:sub".to_vec(), + keys: vec![b":key".to_vec()], + child_info: info.0.to_vec(), + child_type: info.1, + retry_count: None, + }; + issue_request(Request::ReadChild { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_header_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteHeaderRequest { + cht_root: Default::default(), + block: 1, + retry_count: None, + }; + issue_request(Request::Header { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + #[test] + fn receives_remote_changes_response() { + let mut chan = oneshot::channel(); + let request = fetcher::RemoteChangesRequest { + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), + }], + first_block: (1, Default::default()), + last_block: (100, Default::default()), + max_block: (100, Default::default()), + tries_roots: (1, Default::default(), Vec::new()), + key: Vec::new(), + storage_key: None, + retry_count: None, + }; + issue_request(Request::Changes { request, sender: chan.0 }); + assert_matches!(chan.1.try_recv(), Ok(Some(Ok(_)))) + } + + fn send_receive(request: Request) { + // We start a swarm on the listening side which awaits incoming requests and answers them: + let local_pset = peerset(); + let local_listen_addr: libp2p::Multiaddr = libp2p::multiaddr::Protocol::Memory(rand::random()).into(); + let mut local_swarm = make_swarm(true, local_pset.1, make_config()); + Swarm::listen_on(&mut local_swarm, local_listen_addr.clone()).unwrap(); + + // We also start a swarm that makes requests and awaits responses: + let remote_pset = peerset(); + let mut remote_swarm = make_swarm(true, remote_pset.1, make_config()); + + // We now schedule a request, dial the remote and let the two swarm work it out: + remote_swarm.request(request).unwrap(); + Swarm::dial_addr(&mut remote_swarm, local_listen_addr).unwrap(); + + let future = { + let a = local_swarm.for_each(|_| future::ready(())); + let b = remote_swarm.for_each(|_| future::ready(())); + future::join(a, b).map(|_| ()) + }; + + task::spawn(future); + } + + #[test] + fn send_receive_call() { + let chan = oneshot::channel(); + let request = fetcher::RemoteCallRequest { + block: Default::default(), + header: dummy_header(), + method: "test".into(), + call_data: vec![], + retry_count: None, + }; + send_receive(Request::Call { request, sender: chan.0 }); + assert_eq!(vec![42], task::block_on(chan.1).unwrap().unwrap()); + // ^--- from `DummyFetchChecker::check_execution_proof` + } + + #[test] + fn send_receive_read() { + let chan = oneshot::channel(); + let request = fetcher::RemoteReadRequest { + header: dummy_header(), + block: Default::default(), + keys: vec![b":key".to_vec()], + retry_count: None + }; + send_receive(Request::Read { request, sender: chan.0 }); + assert_eq!(Some(vec![42]), task::block_on(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()); + // ^--- from `DummyFetchChecker::check_read_proof` + } + + #[test] + fn send_receive_read_child() { + let info = CHILD_INFO.info(); + let chan = oneshot::channel(); + let request = fetcher::RemoteReadChildRequest { + header: dummy_header(), + block: Default::default(), + storage_key: b":child_storage:sub".to_vec(), + keys: vec![b":key".to_vec()], + child_info: info.0.to_vec(), + child_type: info.1, + retry_count: None, + }; + send_receive(Request::ReadChild { request, sender: chan.0 }); + assert_eq!(Some(vec![42]), task::block_on(chan.1).unwrap().unwrap().remove(&b":key"[..]).unwrap()); + // ^--- from `DummyFetchChecker::check_read_child_proof` + } + + #[test] + fn send_receive_header() { + let _ = env_logger::try_init(); + let chan = oneshot::channel(); + let request = fetcher::RemoteHeaderRequest { + cht_root: Default::default(), + block: 1, + retry_count: None, + }; + send_receive(Request::Header { request, sender: chan.0 }); + // The remote does not know block 1: + assert_matches!(task::block_on(chan.1).unwrap(), Err(ClientError::RemoteFetchFailed)); + } + + #[test] + fn send_receive_changes() { + let chan = oneshot::channel(); + let request = fetcher::RemoteChangesRequest { + changes_trie_configs: vec![sp_core::ChangesTrieConfigurationRange { + zero: (0, Default::default()), + end: None, + config: Some(sp_core::ChangesTrieConfiguration::new(4, 2)), + }], + first_block: (1, Default::default()), + last_block: (100, Default::default()), + max_block: (100, Default::default()), + tries_roots: (1, Default::default(), Vec::new()), + key: Vec::new(), + storage_key: None, + retry_count: None, + }; + send_receive(Request::Changes { request, sender: chan.0 }); + assert_eq!(vec![(100, 2)], task::block_on(chan.1).unwrap().unwrap()); + // ^--- from `DummyFetchChecker::check_changes_proof` + } +} diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index bfa8daa181..2eab0a5a3f 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -21,7 +21,8 @@ use std::collections::{HashMap, VecDeque}; use std::sync::Arc; -use std::time::{Instant, Duration}; +use std::time::Duration; +use wasm_timer::Instant; use log::{trace, info}; use futures::channel::oneshot::{Sender as OneShotSender}; use linked_hash_map::{Entry, LinkedHashMap}; @@ -40,7 +41,7 @@ const REQUEST_TIMEOUT: Duration = Duration::from_secs(15); /// Default request retry count. const RETRY_COUNT: usize = 1; /// Reputation change for a peer when a request timed out. -const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); +pub(crate) const TIMEOUT_REPUTATION_CHANGE: i32 = -(1 << 8); /// Trait used by the `LightDispatch` service to communicate messages back to the network. pub trait LightDispatchNetwork { @@ -691,17 +692,26 @@ pub mod tests { use crate::message::{self, BlockAttributes, Direction, FromBlock, RequestId}; use libp2p::PeerId; use super::{REQUEST_TIMEOUT, LightDispatch, LightDispatchNetwork, RequestData, StorageProof}; - use sp_test_primitives::{Block, Extrinsic, Header}; + use sp_test_primitives::{Block, Header}; - struct DummyFetchChecker { ok: bool } + pub(crate) struct DummyFetchChecker { + pub(crate) ok: bool, + _mark: std::marker::PhantomData + } + + impl DummyFetchChecker { + pub(crate) fn new(ok: bool) -> Self { + DummyFetchChecker { ok, _mark: std::marker::PhantomData } + } + } - impl FetchChecker for DummyFetchChecker { + impl FetchChecker for DummyFetchChecker { fn check_header_proof( &self, - _request: &RemoteHeaderRequest
, - header: Option
, + _request: &RemoteHeaderRequest, + header: Option, _remote_proof: StorageProof, - ) -> ClientResult
{ + ) -> ClientResult { match self.ok { true if header.is_some() => Ok(header.unwrap()), _ => Err(ClientError::Backend("Test error".into())), @@ -710,7 +720,7 @@ pub mod tests { fn check_read_proof( &self, - request: &RemoteReadRequest
, + request: &RemoteReadRequest, _: StorageProof, ) -> ClientResult, Option>>> { match self.ok { @@ -726,7 +736,7 @@ pub mod tests { fn check_read_child_proof( &self, - request: &RemoteReadChildRequest
, + request: &RemoteReadChildRequest, _: StorageProof, ) -> ClientResult, Option>>> { match self.ok { @@ -740,7 +750,7 @@ pub mod tests { } } - fn check_execution_proof(&self, _: &RemoteCallRequest
, _: StorageProof) -> ClientResult> { + fn check_execution_proof(&self, _: &RemoteCallRequest, _: StorageProof) -> ClientResult> { match self.ok { true => Ok(vec![42]), false => Err(ClientError::Backend("Test error".into())), @@ -749,20 +759,20 @@ pub mod tests { fn check_changes_proof( &self, - _: &RemoteChangesRequest
, - _: ChangesProof
- ) -> ClientResult, u32)>> { + _: &RemoteChangesRequest, + _: ChangesProof + ) -> ClientResult, u32)>> { match self.ok { - true => Ok(vec![(100, 2)]), + true => Ok(vec![(100.into(), 2)]), false => Err(ClientError::Backend("Test error".into())), } } fn check_body_proof( &self, - _: &RemoteBodyRequest
, - body: Vec - ) -> ClientResult> { + _: &RemoteBodyRequest, + body: Vec + ) -> ClientResult> { match self.ok { true => Ok(body), false => Err(ClientError::Backend("Test error".into())), @@ -771,7 +781,7 @@ pub mod tests { } fn dummy(ok: bool) -> LightDispatch { - LightDispatch::new(Arc::new(DummyFetchChecker { ok })) + LightDispatch::new(Arc::new(DummyFetchChecker::new(ok))) } fn total_peers(light_dispatch: &LightDispatch) -> usize { @@ -790,7 +800,7 @@ pub mod tests { }); } - fn dummy_header() -> Header { + pub(crate) fn dummy_header() -> Header { Header { parent_hash: Default::default(), number: 0, diff --git a/client/network/src/protocol/schema/api.v1.proto b/client/network/src/protocol/schema/api.v1.proto new file mode 100644 index 0000000000..e4c32ec585 --- /dev/null +++ b/client/network/src/protocol/schema/api.v1.proto @@ -0,0 +1,59 @@ +// Schema definition for block request/response messages. + +syntax = "proto3"; + +package api.v1; + +// Block enumeration direction. +enum Direction { + // Enumerate in ascending order (from child to parent). + Ascending = 0; + // Enumerate in descendfing order (from parent to canonical child). + Descending = 1; +} + +// Request block data from a peer. +message BlockRequest { + // Unique request id. + uint64 id = 1; + // Bits of block data to request. + uint32 fields = 2; + // Start from this block. + oneof from_block { + // Start with given hash. + bytes hash = 3; + // Start with given block number. + bytes number = 4; + } + // End at this block. An implementation defined maximum is used when unspecified. + bytes to_block = 5; // optional + // Sequence direction. + Direction direction = 6; + // Maximum number of blocks to return. An implementation defined maximum is used when unspecified. + uint32 max_blocks = 7; // optional +} + +// Response to `BlockRequest` +message BlockResponse { + // Id of a request this response was made for. + uint64 id = 1; + // Block data for the requested sequence. + repeated BlockData blocks = 2; +} + +// Block data sent in the response. +message BlockData { + // Block header hash. + bytes hash = 1; + // Block header if requested. + bytes header = 2; // optional + // Block body if requested. + repeated bytes body = 3; // optional + // Block receipt if requested. + bytes receipt = 4; // optional + // Block message queue if requested. + bytes message_queue = 5; // optional + // Justification if requested. + bytes justification = 6; // optional +} + diff --git a/client/network/src/protocol/schema/light.v1.proto b/client/network/src/protocol/schema/light.v1.proto new file mode 100644 index 0000000000..b9aee67b5e --- /dev/null +++ b/client/network/src/protocol/schema/light.v1.proto @@ -0,0 +1,128 @@ +// Schema definition for light client messages. + +syntax = "proto3"; + +package api.v1.light; + +// A pair of arbitrary bytes. +message Pair { + // The first element of the pair. + bytes fst = 1; + // The second element of the pair. + bytes snd = 2; +} + +// Enumerate all possible light client request messages. +message Request { + // Unique request id. + uint64 id = 1; + oneof request { + RemoteCallRequest remote_call_request = 2; + RemoteReadRequest remote_read_request = 3; + RemoteHeaderRequest remote_header_request = 4; + RemoteReadChildRequest remote_read_child_request = 5; + RemoteChangesRequest remote_changes_request = 6; + } +} + +// Enumerate all possible light client response messages. +message Response { + /// Id of a request this response was made for. + uint64 id = 1; + oneof response { + RemoteCallResponse remote_call_response = 2; + RemoteReadResponse remote_read_response = 3; + RemoteHeaderResponse remote_header_response = 4; + RemoteChangesResponse remote_changes_response = 6; + } +} + +// Remote call request. +message RemoteCallRequest { + // Block at which to perform call. + bytes block = 2; + // Method name. + string method = 3; + // Call data. + bytes data = 4; +} + +// Remote call response. +message RemoteCallResponse { + // Execution proof. + bytes proof = 2; +} + +// Remote storage read request. +message RemoteReadRequest { + // Block at which to perform call. + bytes block = 2; + // Storage keys. + repeated bytes keys = 3; +} + +// Remote read response. +message RemoteReadResponse { + // Read proof. + bytes proof = 2; +} + +// Remote storage read child request. +message RemoteReadChildRequest { + // Block at which to perform call. + bytes block = 2; + // Child Storage key. + bytes storage_key = 3; + // Child trie source information. + bytes child_info = 4; + /// Child type, its required to resolve `child_info` + /// content and choose child implementation. + uint32 child_type = 5; + // Storage keys. + repeated bytes keys = 6; +} + +// Remote header request. +message RemoteHeaderRequest { + // Block number to request header for. + bytes block = 2; +} + +// Remote header response. +message RemoteHeaderResponse { + // Header. None if proof generation has failed (e.g. header is unknown). + bytes header = 2; // optional + // Header proof. + bytes proof = 3; +} + +/// Remote changes request. +message RemoteChangesRequest { + // Hash of the first block of the range (including first) where changes are requested. + bytes first = 2; + // Hash of the last block of the range (including last) where changes are requested. + bytes last = 3; + // Hash of the first block for which the requester has the changes trie root. All other + // affected roots must be proved. + bytes min = 4; + // Hash of the last block that we can use when querying changes. + bytes max = 5; + // Storage child node key which changes are requested. + bytes storage_key = 6; // optional + // Storage key which changes are requested. + bytes key = 7; +} + +// Remote changes response. +message RemoteChangesResponse { + // Proof has been generated using block with this number as a max block. Should be + // less than or equal to the RemoteChangesRequest::max block number. + bytes max = 2; + // Changes proof. + repeated bytes proof = 3; + // Changes tries roots missing on the requester' node. + repeated Pair roots = 4; + // Missing changes tries roots proof. + bytes roots_proof = 5; +} + diff --git a/client/network/src/protocol/sync/extra_requests.rs b/client/network/src/protocol/sync/extra_requests.rs index 44b42b154f..a5fb232b8d 100644 --- a/client/network/src/protocol/sync/extra_requests.rs +++ b/client/network/src/protocol/sync/extra_requests.rs @@ -21,7 +21,8 @@ use libp2p::PeerId; use log::{debug, trace, warn}; use sp_runtime::traits::{Block as BlockT, NumberFor, Zero}; use std::collections::{HashMap, HashSet, VecDeque}; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; // Time to wait before trying to get the same extra data from the same peer. const EXTRA_RETRY_WAIT: Duration = Duration::from_secs(10); diff --git a/client/network/src/service.rs b/client/network/src/service.rs index e3eaf6c31d..156e2e3102 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -193,6 +193,10 @@ impl, H: ExHashT> NetworkWorker let local_peer_id = local_public.clone().into_peer_id(); info!(target: "sub-libp2p", "Local node identity is: {}", local_peer_id.to_base58()); + let checker = params.on_demand.as_ref() + .map(|od| od.checker().clone()) + .unwrap_or(Arc::new(AlwaysBadChecker)); + let num_connected = Arc::new(AtomicUsize::new(0)); let is_major_syncing = Arc::new(AtomicBool::new(false)); let (protocol, peerset_handle) = Protocol::new( @@ -200,14 +204,13 @@ impl, H: ExHashT> NetworkWorker roles: params.roles, max_parallel_downloads: params.network_config.max_parallel_downloads, }, - params.chain, - params.on_demand.as_ref().map(|od| od.checker().clone()) - .unwrap_or(Arc::new(AlwaysBadChecker)), + params.chain.clone(), + checker.clone(), params.specialization, params.transaction_pool, - params.finality_proof_provider, + params.finality_proof_provider.clone(), params.finality_proof_request_builder, - params.protocol_id, + params.protocol_id.clone(), peerset_config, params.block_announce_validator )?; @@ -219,6 +222,14 @@ impl, H: ExHashT> NetworkWorker params.network_config.client_version, params.network_config.node_name ); + let block_requests = { + let config = protocol::block_requests::Config::new(¶ms.protocol_id); + protocol::BlockRequests::new(config, params.chain.clone()) + }; + let light_client_handler = { + let config = protocol::light_client_handler::Config::new(¶ms.protocol_id); + protocol::LightClientHandler::new(config, params.chain, checker, peerset_handle.clone()) + }; let behaviour = futures::executor::block_on(Behaviour::new( protocol, user_agent, @@ -232,14 +243,17 @@ impl, H: ExHashT> NetworkWorker TransportConfig::MemoryOnly => false, TransportConfig::Normal { allow_private_ipv4, .. } => allow_private_ipv4, }, + u64::from(params.network_config.out_peers) + 15, + block_requests, + light_client_handler )); let (transport, bandwidth) = { - let (config_mem, config_wasm) = match params.network_config.transport { - TransportConfig::MemoryOnly => (true, None), - TransportConfig::Normal { wasm_external_transport, .. } => - (false, wasm_external_transport) + let (config_mem, config_wasm, flowctrl) = match params.network_config.transport { + TransportConfig::MemoryOnly => (true, None, false), + TransportConfig::Normal { wasm_external_transport, use_yamux_flow_control, .. } => + (false, wasm_external_transport, use_yamux_flow_control) }; - transport::build_transport(local_identity, config_mem, config_wasm) + transport::build_transport(local_identity, config_mem, config_wasm, flowctrl) }; let mut builder = SwarmBuilder::new(transport, behaviour, local_peer_id.clone()); if let Some(spawner) = params.executor { diff --git a/client/network/src/transport.rs b/client/network/src/transport.rs index b11b187051..e2c95824f8 100644 --- a/client/network/src/transport.rs +++ b/client/network/src/transport.rs @@ -17,7 +17,7 @@ use futures::prelude::*; use libp2p::{ InboundUpgradeExt, OutboundUpgradeExt, PeerId, Transport, - mplex, identity, yamux, bandwidth, wasm_ext + mplex, identity, bandwidth, wasm_ext }; #[cfg(not(target_os = "unknown"))] use libp2p::{tcp, dns, websocket, noise}; @@ -36,7 +36,8 @@ pub use self::bandwidth::BandwidthSinks; pub fn build_transport( keypair: identity::Keypair, memory_only: bool, - wasm_external_transport: Option + wasm_external_transport: Option, + use_yamux_flow_control: bool ) -> (Boxed<(PeerId, StreamMuxerBox), io::Error>, Arc) { // Build configuration objects for encryption mechanisms. #[cfg(not(target_os = "unknown"))] @@ -55,7 +56,18 @@ pub fn build_transport( let mut mplex_config = mplex::MplexConfig::new(); mplex_config.max_buffer_len_behaviour(mplex::MaxBufferBehaviour::Block); mplex_config.max_buffer_len(usize::MAX); - let yamux_config = yamux::Config::default(); + + let yamux_config = { + let mut c = yamux::Config::default(); + // Only set SYN flag on first data frame sent to the remote. + c.set_lazy_open(true); + if use_yamux_flow_control { + // Enable proper flow-control: window updates are only sent when + // buffered data has been consumed. + c.set_window_update_mode(yamux::WindowUpdateMode::OnRead); + } + libp2p::yamux::Config::new(c) + }; // Build the base layer of the transport. let transport = if let Some(t) = wasm_external_transport { @@ -118,11 +130,18 @@ pub fn build_transport( core::upgrade::apply(stream, upgrade, endpoint, upgrade::Version::V1) .map_ok(|(id, muxer)| (id, core::muxing::StreamMuxerBox::new(muxer))) - }) + }); - .timeout(Duration::from_secs(20)) - .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) - .boxed(); + let transport = if cfg!(not(target_os = "unknown")) { + transport + .timeout(Duration::from_secs(20)) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) + .boxed() + } else { + transport + .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) + .boxed() + }; (transport, sinks) } diff --git a/client/network/test/Cargo.toml b/client/network/test/Cargo.toml index 3f1d1d5867..ca4b0237cf 100644 --- a/client/network/test/Cargo.toml +++ b/client/network/test/Cargo.toml @@ -12,7 +12,7 @@ log = "0.4.8" parking_lot = "0.10.0" futures = "0.1.29" futures03 = { package = "futures", version = "0.3.1", features = ["compat"] } -futures-timer = "0.4.0" +futures-timer = "3.0.1" rand = "0.7.2" libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } sp-consensus = { version = "0.8", path = "../../../primitives/consensus/common" } diff --git a/client/network/test/src/sync.rs b/client/network/test/src/sync.rs index 2140de0973..210a4fb38b 100644 --- a/client/network/test/src/sync.rs +++ b/client/network/test/src/sync.rs @@ -405,7 +405,7 @@ fn blocks_are_not_announced_by_light_nodes() { net.peers.remove(0); // Poll for a few seconds and make sure 1 and 2 (now 0 and 1) don't sync together. - let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).compat(); + let mut delay = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); runtime.block_on(futures::future::poll_fn::<(), (), _>(|| { net.poll(); delay.poll().map_err(|_| ()) @@ -504,7 +504,7 @@ fn can_not_sync_from_light_peer() { net.peers.remove(0); // ensure that the #2 (now #1) fails to sync block #1 even after 5 seconds - let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).compat(); + let mut test_finished = futures_timer::Delay::new(Duration::from_secs(5)).unit_error().compat(); runtime.block_on(futures::future::poll_fn::<(), (), _>(|| -> Result<_, ()> { net.poll(); test_finished.poll().map_err(|_| ()) diff --git a/client/offchain/Cargo.toml b/client/offchain/Cargo.toml index bdbf3fa199..ed5db3f2c7 100644 --- a/client/offchain/Cargo.toml +++ b/client/offchain/Cargo.toml @@ -13,7 +13,7 @@ sp-api = { version = "2.0.0", path = "../../primitives/api" } fnv = "1.0.6" futures01 = { package = "futures", version = "0.1" } futures = "0.3.1" -futures-timer = "2.0" +futures-timer = "3.0.1" log = "0.4.8" threadpool = "1.7" num_cpus = "1.10" diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index f0a10a5409..fadfaa0134 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -97,7 +97,7 @@ impl OffchainWorkers< is_validator: bool, ) -> impl Future { let runtime = self.client.runtime_api(); - let at = BlockId::number(*header.number()); + let at = BlockId::hash(header.hash()); let has_api_v1 = runtime.has_api_with::, _>( &at, |v| v == 1 ); @@ -169,7 +169,6 @@ mod tests { use substrate_test_runtime_client::runtime::Block; use sc_transaction_pool::{BasicPool, FullChainApi}; use sp_transaction_pool::{TransactionPool, InPoolTransaction}; - use sp_runtime::{generic::Header, traits::Header as _}; struct MockNetworkStateInfo(); @@ -210,13 +209,7 @@ mod tests { .register_transaction_pool(Arc::downgrade(&pool.clone()) as _); let db = sc_client_db::offchain::LocalStorage::new_test(); let network_state = Arc::new(MockNetworkStateInfo()); - let header = Header::new( - 0u64, - Default::default(), - Default::default(), - Default::default(), - Default::default(), - ); + let header = client.header(&BlockId::number(0)).unwrap().unwrap(); // when let offchain = OffchainWorkers::new(client, db); diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 85c988d048..2527b86795 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -12,6 +12,7 @@ futures = "0.3.1" libp2p = { version = "0.15.0", default-features = false } log = "0.4.8" serde_json = "1.0.41" +wasm-timer = "0.2" [dev-dependencies] rand = "0.7.2" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index c958912727..bd2b6bb110 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -19,17 +19,18 @@ mod peersstate; -use std::{collections::{HashSet, HashMap}, collections::VecDeque, time::Instant}; +use std::{collections::{HashSet, HashMap}, collections::VecDeque}; use futures::{prelude::*, channel::mpsc}; use libp2p::PeerId; use log::{debug, error, trace}; use serde_json::json; use std::{pin::Pin, task::Context, task::Poll}; +use wasm_timer::Instant; /// We don't accept nodes whose reputation is under this value. const BANNED_THRESHOLD: i32 = 82 * (i32::min_value() / 100); /// Reputation change for a node when we get disconnected from it. -const DISCONNECT_REPUTATION_CHANGE: i32 = -10; +const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// Reserved peers group ID const RESERVED_NODES: &'static str = "reserved"; diff --git a/client/rpc-api/Cargo.toml b/client/rpc-api/Cargo.toml index 4781c9d350..b9f4cbb015 100644 --- a/client/rpc-api/Cargo.toml +++ b/client/rpc-api/Cargo.toml @@ -17,6 +17,7 @@ log = "0.4.8" parking_lot = "0.10.0" sp-core = { version = "2.0.0", path = "../../primitives/core" } sp-version = { version = "2.0.0", path = "../../primitives/version" } +sp-runtime = { path = "../../primitives/runtime" } serde = { version = "1.0.101", features = ["derive"] } serde_json = "1.0.41" sp-transaction-pool = { version = "2.0.0", path = "../../primitives/transaction-pool" } diff --git a/client/service/Cargo.toml b/client/service/Cargo.toml index bc246db03e..7f7ef80b16 100644 --- a/client/service/Cargo.toml +++ b/client/service/Cargo.toml @@ -24,7 +24,8 @@ lazy_static = "1.4.0" log = "0.4.8" slog = { version = "2.5.2", features = ["nested-values"] } tokio-executor = "0.1.8" -futures-timer = "2" +futures-timer = "3.0.1" +wasm-timer = "0.2" exit-future = "0.2.0" serde = "1.0.101" serde_json = "1.0.41" diff --git a/client/service/src/builder.rs b/client/service/src/builder.rs index 2c10804675..5ca39856dc 100644 --- a/client/service/src/builder.rs +++ b/client/service/src/builder.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID}; +use crate::{Service, NetworkStatus, NetworkState, error::Error, DEFAULT_PROTOCOL_ID, MallocSizeOfWasm}; use crate::{SpawnTaskHandle, start_rpc_servers, build_network_future, TransactionPoolAdapter}; use crate::status_sinks; use crate::config::{Configuration, DatabaseConfig, KeystoreConfig}; @@ -46,11 +46,12 @@ use sc_executor::{NativeExecutor, NativeExecutionDispatch}; use std::{ borrow::Cow, io::{Read, Write, Seek}, - marker::PhantomData, sync::Arc, time::SystemTime, pin::Pin + marker::PhantomData, sync::Arc, pin::Pin }; +use wasm_timer::SystemTime; use sysinfo::{get_current_pid, ProcessExt, System, SystemExt}; use sc_telemetry::{telemetry, SUBSTRATE_INFO}; -use sp_transaction_pool::MaintainedTransactionPool; +use sp_transaction_pool::{MaintainedTransactionPool, ChainEvent}; use sp_blockchain; use grafana_data_source::{self, record_metrics}; @@ -673,6 +674,8 @@ impl, TNetP: NetworkSpecialization, - TExPool: MaintainedTransactionPool::Hash> + 'static, + TExPool: MaintainedTransactionPool::Hash> + MallocSizeOfWasm + 'static, TRpc: sc_rpc::RpcExtension + Clone, { @@ -879,34 +882,52 @@ ServiceBuilder< let network_state_info: Arc = network.clone(); let is_validator = config.roles.is_authority(); - let events = client.import_notification_stream() - .for_each(move |notification| { - let txpool = txpool.upgrade(); + let (import_stream, finality_stream) = ( + client.import_notification_stream().map(|n| ChainEvent::NewBlock { + id: BlockId::Hash(n.hash), + header: n.header, + retracted: n.retracted, + is_new_best: n.is_new_best, + }), + client.finality_notification_stream().map(|n| ChainEvent::Finalized { + hash: n.hash + }) + ); + let events = futures::stream::select(import_stream, finality_stream) + .for_each(move |event| { + // offchain worker is only interested in block import events + if let ChainEvent::NewBlock { ref header, is_new_best, .. } = event { + let offchain = offchain.as_ref().and_then(|o| o.upgrade()); + match offchain { + Some(offchain) if is_new_best => { + let future = offchain.on_block_imported( + &header, + network_state_info.clone(), + is_validator, + ); + let _ = to_spawn_tx_.unbounded_send(( + Box::pin(future), + From::from("offchain-on-block"), + )); + }, + Some(_) => log::debug!( + target: "sc_offchain", + "Skipping offchain workers for non-canon block: {:?}", + header, + ), + _ => {}, + } + }; + let txpool = txpool.upgrade(); if let Some(txpool) = txpool.as_ref() { - let future = txpool.maintain( - &BlockId::hash(notification.hash), - ¬ification.retracted, - ); + let future = txpool.maintain(event); let _ = to_spawn_tx_.unbounded_send(( Box::pin(future), From::from("txpool-maintain") )); } - let offchain = offchain.as_ref().and_then(|o| o.upgrade()); - if let Some(offchain) = offchain { - let future = offchain.on_block_imported( - ¬ification.header, - network_state_info.clone(), - is_validator - ); - let _ = to_spawn_tx_.unbounded_send(( - Box::pin(future), - From::from("offchain-on-block") - )); - } - ready(()) }); let _ = to_spawn_tx.unbounded_send(( @@ -982,6 +1003,10 @@ ServiceBuilder< "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), ); + #[cfg(not(target_os = "unknown"))] + let memory_transaction_pool = parity_util_mem::malloc_size(&*transaction_pool_); + #[cfg(target_os = "unknown")] + let memory_transaction_pool = 0; let _ = record_metrics!( "peers" => num_peers, "height" => best_number, @@ -995,7 +1020,7 @@ ServiceBuilder< "used_db_cache_size" => info.usage.as_ref().map(|usage| usage.memory.database_cache).unwrap_or(0), "disk_read_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_read).unwrap_or(0), "disk_write_per_sec" => info.usage.as_ref().map(|usage| usage.io.bytes_written).unwrap_or(0), - "memory_transaction_pool" => parity_util_mem::malloc_size(&*transaction_pool_), + "memory_transaction_pool" => memory_transaction_pool, ); ready(()) diff --git a/client/service/src/chain_ops.rs b/client/service/src/chain_ops.rs index cac1c14d0b..3d77d9c815 100644 --- a/client/service/src/chain_ops.rs +++ b/client/service/src/chain_ops.rs @@ -22,14 +22,20 @@ use crate::error::Error; use sc_chain_spec::{ChainSpec, RuntimeGenesis, Extension}; use log::{warn, info}; use futures::{future, prelude::*}; -use sp_runtime::traits::{ - Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion +use sp_runtime::{ + BuildStorage, BenchmarkResults, + traits::{ + Block as BlockT, NumberFor, One, Zero, Header, SaturatedConversion + } }; use sp_runtime::generic::{BlockId, SignedBlock}; use codec::{Decode, Encode, IoReader}; -use sc_client::Client; +use sc_client::{Client, ExecutionStrategy, StateMachine, LocalCallExecutor}; +#[cfg(feature = "rocksdb")] +use sc_client_db::BenchmarkingState; use sp_consensus::import_queue::{IncomingBlock, Link, BlockImportError, BlockImportResult, ImportQueue}; use sp_consensus::BlockOrigin; +use sc_executor::{NativeExecutor, NativeExecutionDispatch, WasmExecutionMethod}; use std::{io::{Read, Write, Seek}, pin::Pin}; @@ -43,21 +49,76 @@ pub fn build_spec(spec: ChainSpec, raw: bool) -> error::Result ( + spec: ChainSpec, + strategy: ExecutionStrategy, + wasm_method: WasmExecutionMethod, + pallet: String, + extrinsic: String, + steps: u32, + repeat: u32, +) -> error::Result<()> where + TBl: BlockT, + TExecDisp: NativeExecutionDispatch + 'static, + G: RuntimeGenesis, + E: Extension, +{ + let genesis_storage = spec.build_storage()?; + let mut changes = Default::default(); + let state = BenchmarkingState::::new(genesis_storage)?; + let executor = NativeExecutor::::new( + wasm_method, + None, // heap pages + ); + let result = StateMachine::<_, _, NumberFor, _>::new( + &state, + None, + &mut changes, + &executor, + "Benchmark_dispatch_benchmark", + &(&pallet, &extrinsic, steps, repeat).encode(), + Default::default(), + ).execute(strategy).map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?; + let results = > as Decode>::decode(&mut &result[..]).unwrap_or(None); + if let Some(results) = results { + // Print benchmark metadata + println!("Pallet: {:?}, Extrinsic: {:?}, Steps: {:?}, Repeat: {:?}", pallet, extrinsic, steps, repeat); + // Print the table header + results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + print!("time\n"); + // Print the values + results.iter().for_each(|result| { + let parameters = &result.0; + parameters.iter().for_each(|param| print!("{:?},", param.1)); + print!("{:?}\n", result.1); + }); + info!("Done."); + } else { + info!("No Results."); + } + Ok(()) +} + + impl< TBl, TRtApi, TGen, TCSExt, TBackend, - TExec, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, + TExecDisp, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TExPool, TRpc, Backend > ServiceBuilderCommand for ServiceBuilder< - TBl, TRtApi, TGen, TCSExt, Client, + TBl, TRtApi, TGen, TCSExt, + Client>, TBl, TRtApi>, TFchr, TSc, TImpQu, TFprb, TFpp, TNetP, TExPool, TRpc, Backend > where TBl: BlockT, TBackend: 'static + sc_client_api::backend::Backend + Send, - TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, + TExecDisp: 'static + NativeExecutionDispatch, TImpQu: 'static + ImportQueue, TRtApi: 'static + Send + Sync, { type Block = TBl; + type NativeDispatch = TExecDisp; fn import_blocks( self, diff --git a/client/service/src/lib.rs b/client/service/src/lib.rs index 577f36572a..8c2f57dd7c 100644 --- a/client/service/src/lib.rs +++ b/client/service/src/lib.rs @@ -31,7 +31,8 @@ use std::{borrow::Cow, io, pin::Pin}; use std::marker::PhantomData; use std::net::SocketAddr; use std::collections::HashMap; -use std::time::{Duration, Instant}; +use std::time::Duration; +use wasm_timer::Instant; use std::task::{Poll, Context}; use parking_lot::Mutex; @@ -52,6 +53,7 @@ use log::{log, warn, debug, error, Level}; use codec::{Encode, Decode}; use sp_runtime::generic::BlockId; use sp_runtime::traits::{NumberFor, Block as BlockT}; +use parity_util_mem::MallocSizeOf; pub use self::error::Error; pub use self::builder::{ @@ -65,6 +67,7 @@ pub use sp_transaction_pool::{TransactionPool, InPoolTransaction, error::IntoPoo pub use sc_transaction_pool::txpool::Options as TransactionPoolOptions; pub use sc_client::FinalityNotifications; pub use sc_rpc::Metadata as RpcMetadata; +pub use sc_executor::NativeExecutionDispatch; #[doc(hidden)] pub use std::{ops::Deref, result::Result, sync::Arc}; #[doc(hidden)] @@ -72,6 +75,16 @@ pub use sc_network::{FinalityProofProvider, OnDemand, config::BoxFinalityProofRe const DEFAULT_PROTOCOL_ID: &str = "sup"; +/// A type that implements `MallocSizeOf` on native but not wasm. +#[cfg(not(target_os = "unknown"))] +pub trait MallocSizeOfWasm: MallocSizeOf {} +#[cfg(target_os = "unknown")] +pub trait MallocSizeOfWasm {} +#[cfg(not(target_os = "unknown"))] +impl MallocSizeOfWasm for T {} +#[cfg(target_os = "unknown")] +impl MallocSizeOfWasm for T {} + /// Substrate service. pub struct Service { client: Arc, @@ -162,7 +175,7 @@ pub trait AbstractService: 'static + Future> + /// Chain selection algorithm. type SelectChain: sp_consensus::SelectChain; /// Transaction pool. - type TransactionPool: TransactionPool; + type TransactionPool: TransactionPool + MallocSizeOfWasm; /// Network specialization. type NetworkSpecialization: NetworkSpecialization; @@ -226,7 +239,7 @@ where TExec: 'static + sc_client::CallExecutor + Send + Sync + Clone, TRtApi: 'static + Send + Sync, TSc: sp_consensus::SelectChain + 'static + Clone + Send + Unpin, - TExPool: 'static + TransactionPool, + TExPool: 'static + TransactionPool + MallocSizeOfWasm, TOc: 'static + Send + Sync, TNetSpec: NetworkSpecialization, { diff --git a/client/service/test/src/lib.rs b/client/service/test/src/lib.rs index 2976e66a29..723c13ec82 100644 --- a/client/service/test/src/lib.rs +++ b/client/service/test/src/lib.rs @@ -166,6 +166,7 @@ fn node_config ( enable_mdns: false, allow_private_ipv4: true, wasm_external_transport: None, + use_yamux_flow_control: true, }, max_parallel_downloads: NetworkConfiguration::default().max_parallel_downloads, }; diff --git a/client/src/cht.rs b/client/src/cht.rs index 29f19a7750..1435b77ec5 100644 --- a/client/src/cht.rs +++ b/client/src/cht.rs @@ -28,7 +28,7 @@ use codec::Encode; use sp_trie; use sp_core::{H256, convert_hash}; -use sp_runtime::traits::{Header as HeaderT, SimpleArithmetic, Zero, One}; +use sp_runtime::traits::{Header as HeaderT, AtLeast32Bit, Zero, One}; use sp_state_machine::{ MemoryDB, TrieBackend, Backend as StateBackend, StorageProof, InMemoryBackend, prove_read_on_trie_backend, read_proof_check, read_proof_check_on_proving_backend @@ -48,7 +48,7 @@ pub fn size>() -> N { /// Returns Some(cht_number) if CHT is need to be built when the block with given number is canonized. pub fn is_build_required(cht_size: N, block_num: N) -> Option where - N: Clone + SimpleArithmetic, + N: Clone + AtLeast32Bit, { let block_cht_num = block_to_cht_number(cht_size.clone(), block_num.clone())?; let two = N::one() + N::one(); @@ -66,7 +66,7 @@ pub fn is_build_required(cht_size: N, block_num: N) -> Option /// Returns Some(max_cht_number) if CHT has ever been built given maximal canonical block number. pub fn max_cht_number(cht_size: N, max_canonical_block: N) -> Option where - N: Clone + SimpleArithmetic, + N: Clone + AtLeast32Bit, { let max_cht_number = block_to_cht_number(cht_size, max_canonical_block)?; let two = N::one() + N::one(); @@ -291,18 +291,18 @@ fn build_pairs( /// More generally: CHT N includes block (1 + N*SIZE)...((N+1)*SIZE). /// This is because the genesis hash is assumed to be known /// and including it would be redundant. -pub fn start_number(cht_size: N, cht_num: N) -> N { +pub fn start_number(cht_size: N, cht_num: N) -> N { (cht_num * cht_size) + N::one() } /// Get the ending block of a given CHT. -pub fn end_number(cht_size: N, cht_num: N) -> N { +pub fn end_number(cht_size: N, cht_num: N) -> N { (cht_num + N::one()) * cht_size } /// Convert a block number to a CHT number. /// Returns `None` for `block_num` == 0, `Some` otherwise. -pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { +pub fn block_to_cht_number(cht_size: N, block_num: N) -> Option { if block_num == N::zero() { None } else { diff --git a/client/src/client.rs b/client/src/client.rs index 9e30c7b2ea..d085b92025 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -1599,6 +1599,9 @@ impl sp_consensus::BlockImport for &Client>, new_cache: HashMap>, ) -> Result { + let span = tracing::span!(tracing::Level::DEBUG, "import_block"); + let _enter = span.enter(); + if let Some(res) = self.prepare_block_storage_changes(&mut import_block).map_err(|e| { warn!("Block prepare storage changes error:\n{:?}", e); ConsensusError::ClientImport(e.to_string()) diff --git a/client/src/leaves.rs b/client/src/leaves.rs index bb556da83a..1082e6ca07 100644 --- a/client/src/leaves.rs +++ b/client/src/leaves.rs @@ -19,7 +19,7 @@ use std::collections::BTreeMap; use std::cmp::Reverse; use kvdb::{KeyValueDB, DBTransaction}; -use sp_runtime::traits::SimpleArithmetic; +use sp_runtime::traits::AtLeast32Bit; use codec::{Encode, Decode}; use sp_blockchain::{Error, Result}; @@ -65,7 +65,7 @@ pub struct LeafSet { impl LeafSet where H: Clone + PartialEq + Decode + Encode, - N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode, + N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode, { /// Construct a new, blank leaf set. pub fn new() -> Self { @@ -251,7 +251,7 @@ pub struct Undo<'a, H: 'a, N: 'a> { impl<'a, H: 'a, N: 'a> Undo<'a, H, N> where H: Clone + PartialEq + Decode + Encode, - N: std::fmt::Debug + Clone + SimpleArithmetic + Decode + Encode, + N: std::fmt::Debug + Clone + AtLeast32Bit + Decode + Encode, { /// Undo an imported block by providing the displaced leaf. pub fn undo_import(&mut self, displaced: ImportDisplaced) { diff --git a/client/src/lib.rs b/client/src/lib.rs index 4caabfa201..d97246d478 100644 --- a/client/src/lib.rs +++ b/client/src/lib.rs @@ -104,4 +104,4 @@ pub use crate::{ }, leaves::LeafSet, }; -pub use sp_state_machine::{ExecutionStrategy, StorageProof}; +pub use sp_state_machine::{ExecutionStrategy, StorageProof, StateMachine}; diff --git a/client/src/light/fetcher.rs b/client/src/light/fetcher.rs index d66108b7f0..9df6a38630 100644 --- a/client/src/light/fetcher.rs +++ b/client/src/light/fetcher.rs @@ -25,7 +25,7 @@ use codec::{Decode, Encode}; use sp_core::{convert_hash, traits::CodeExecutor}; use sp_runtime::traits::{ Block as BlockT, Header as HeaderT, Hash, HashFor, NumberFor, - SimpleArithmetic, CheckedConversion, + AtLeast32Bit, CheckedConversion, }; use sp_state_machine::{ ChangesTrieRootsStorage, ChangesTrieAnchorBlockId, ChangesTrieConfigurationRange, @@ -286,7 +286,7 @@ impl FetchChecker for LightDataChecker } /// A view of BTreeMap as a changes trie roots storage. -struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> { +struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { roots: (Number, &'a [Hash]), prev_roots: &'a BTreeMap, } @@ -294,7 +294,7 @@ struct RootsStorage<'a, Number: SimpleArithmetic, Hash: 'a> { impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> where H: Hasher, - Number: ::std::fmt::Display + ::std::hash::Hash + Clone + SimpleArithmetic + Encode + Decode + Send + Sync + 'static, + Number: ::std::fmt::Display + ::std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, { fn build_anchor( diff --git a/client/telemetry/Cargo.toml b/client/telemetry/Cargo.toml index 44e332b9d7..f75c6dd44e 100644 --- a/client/telemetry/Cargo.toml +++ b/client/telemetry/Cargo.toml @@ -10,7 +10,8 @@ license = "GPL-3.0" bytes = "0.5" parking_lot = "0.10.0" futures = "0.3.1" -futures-timer = "2.0.0" +futures-timer = "3.0.1" +wasm-timer = "0.2.0" libp2p = { version = "0.15.0", default-features = false, features = ["libp2p-websocket"] } log = "0.4.8" pin-project = "0.4.6" diff --git a/client/telemetry/src/lib.rs b/client/telemetry/src/lib.rs index 906df545e2..1c6f1425d6 100644 --- a/client/telemetry/src/lib.rs +++ b/client/telemetry/src/lib.rs @@ -63,7 +63,8 @@ use libp2p::{Multiaddr, wasm_ext}; use log::{error, warn}; use parking_lot::Mutex; use serde::{Serialize, Deserialize}; -use std::{pin::Pin, sync::Arc, task::{Context, Poll}, time::{Duration, Instant}}; +use std::{pin::Pin, sync::Arc, task::{Context, Poll}, time::Duration}; +use wasm_timer::Instant; pub use libp2p::wasm_ext::ExtTransport; pub use slog_scope::with_logger; diff --git a/client/telemetry/src/worker.rs b/client/telemetry/src/worker.rs index 8f43bb612a..ef1f9fa067 100644 --- a/client/telemetry/src/worker.rs +++ b/client/telemetry/src/worker.rs @@ -98,10 +98,10 @@ impl TelemetryWorker { .map_ok(|data| BytesMut::from(data.as_ref())); future::ready(Ok::<_, io::Error>(connec)) }) - }); + }) + .timeout(CONNECT_TIMEOUT); let transport = transport - .timeout(CONNECT_TIMEOUT) .map_err(|err| io::Error::new(io::ErrorKind::Other, err)) .map(|out, _| { let out = out diff --git a/client/transaction-pool/Cargo.toml b/client/transaction-pool/Cargo.toml index 524e9a98a0..27aec1b92e 100644 --- a/client/transaction-pool/Cargo.toml +++ b/client/transaction-pool/Cargo.toml @@ -12,6 +12,7 @@ futures = { version = "0.3.1", features = ["compat"] } futures-diagnose = "1.0" log = "0.4.8" parking_lot = "0.10.0" +wasm-timer = "0.2" sp-core = { version = "2.0.0", path = "../../primitives/core" } sp-api = { version = "2.0.0", path = "../../primitives/api" } sp-runtime = { version = "2.0.0", path = "../../primitives/runtime" } diff --git a/client/transaction-pool/graph/Cargo.toml b/client/transaction-pool/graph/Cargo.toml index 2d3172fc91..daec970a69 100644 --- a/client/transaction-pool/graph/Cargo.toml +++ b/client/transaction-pool/graph/Cargo.toml @@ -11,10 +11,13 @@ futures = "0.3.1" log = "0.4.8" parking_lot = "0.10.0" serde = { version = "1.0.101", features = ["derive"] } +wasm-timer = "0.2" +sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } sp-core = { version = "2.0.0", path = "../../../primitives/core" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } parity-util-mem = { version = "0.5.1", default-features = false, features = ["primitive-types"] } +linked-hash-map = "0.5.2" [dev-dependencies] assert_matches = "1.3.0" diff --git a/client/transaction-pool/graph/benches/basics.rs b/client/transaction-pool/graph/benches/basics.rs index f3f67f1446..54bbe930b3 100644 --- a/client/transaction-pool/graph/benches/basics.rs +++ b/client/transaction-pool/graph/benches/basics.rs @@ -135,8 +135,8 @@ fn bench_configured(pool: Pool, number: u64) { let res = block_on(futures::future::join_all(futures.into_iter())); assert!(res.iter().all(Result::is_ok)); - assert_eq!(pool.status().future, 0); - assert_eq!(pool.status().ready, number as usize); + assert_eq!(pool.validated_pool().status().future, 0); + assert_eq!(pool.validated_pool().status().ready, number as usize); // Prune all transactions. let block_num = 6; @@ -147,8 +147,8 @@ fn bench_configured(pool: Pool, number: u64) { )).expect("Prune failed"); // pool is empty - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); } fn benchmark_main(c: &mut Criterion) { diff --git a/client/transaction-pool/graph/src/base_pool.rs b/client/transaction-pool/graph/src/base_pool.rs index 52e00df363..8a33d8244e 100644 --- a/client/transaction-pool/graph/src/base_pool.rs +++ b/client/transaction-pool/graph/src/base_pool.rs @@ -209,7 +209,8 @@ const RECENTLY_PRUNED_TAGS: usize = 2; /// as-is for the second time will fail or produce unwanted results. /// Most likely it is required to revalidate them and recompute set of /// required tags. -#[derive(Debug, parity_util_mem::MallocSizeOf)] +#[derive(Debug)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] pub struct BasePool { reject_future_transactions: bool, future: FutureTransactions, diff --git a/client/transaction-pool/graph/src/future.rs b/client/transaction-pool/graph/src/future.rs index bda26fe34f..a84a5fbe68 100644 --- a/client/transaction-pool/graph/src/future.rs +++ b/client/transaction-pool/graph/src/future.rs @@ -19,17 +19,17 @@ use std::{ fmt, hash, sync::Arc, - time, }; use sp_core::hexdisplay::HexDisplay; use sp_runtime::transaction_validity::{ TransactionTag as Tag, }; +use wasm_timer::Instant; use crate::base_pool::Transaction; -#[derive(parity_util_mem::MallocSizeOf)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] /// Transaction with partially satisfied dependencies. pub struct WaitingTransaction { /// Transaction details. @@ -37,7 +37,7 @@ pub struct WaitingTransaction { /// Tags that are required and have not been satisfied yet by other transactions in the pool. pub missing_tags: HashSet, /// Time of import to the Future Queue. - pub imported_at: time::Instant, + pub imported_at: Instant, } impl fmt::Debug for WaitingTransaction { @@ -91,7 +91,7 @@ impl WaitingTransaction { WaitingTransaction { transaction: Arc::new(transaction), missing_tags, - imported_at: time::Instant::now(), + imported_at: Instant::now(), } } @@ -110,7 +110,8 @@ impl WaitingTransaction { /// /// Contains transactions that are still awaiting for some other transactions that /// could provide a tag that they require. -#[derive(Debug, parity_util_mem::MallocSizeOf)] +#[derive(Debug)] +#[cfg_attr(not(target_os = "unknown"), derive(parity_util_mem::MallocSizeOf))] pub struct FutureTransactions { /// tags that are not yet provided by any transaction and we await for them wanted_tags: HashMap>, diff --git a/client/transaction-pool/graph/src/listener.rs b/client/transaction-pool/graph/src/listener.rs index dab2a6f5aa..be6fb5c990 100644 --- a/client/transaction-pool/graph/src/listener.rs +++ b/client/transaction-pool/graph/src/listener.rs @@ -16,30 +16,34 @@ // along with Substrate. If not, see . use std::{ - collections::HashMap, - fmt, - hash, + collections::HashMap, hash, }; +use linked_hash_map::LinkedHashMap; use serde::Serialize; -use crate::watcher; -use sp_runtime::traits; +use crate::{watcher, ChainApi, BlockHash}; use log::{debug, trace, warn}; +use sp_runtime::traits; /// Extrinsic pool default listener. -pub struct Listener { - watchers: HashMap> +pub struct Listener { + watchers: HashMap>>, + finality_watchers: LinkedHashMap, Vec>, } -impl Default for Listener { +/// Maximum number of blocks awaiting finality at any time. +const MAX_FINALITY_WATCHERS: usize = 512; + +impl Default for Listener { fn default() -> Self { Listener { watchers: Default::default(), + finality_watchers: Default::default(), } } } -impl Listener { - fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender) { +impl Listener { + fn fire(&mut self, hash: &H, fun: F) where F: FnOnce(&mut watcher::Sender>) { let clean = if let Some(h) = self.watchers.get_mut(hash) { fun(h); h.is_done() @@ -55,7 +59,7 @@ impl Listene /// Creates a new watcher for given verified extrinsic. /// /// The watcher can be used to subscribe to lifecycle events of that extrinsic. - pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher { + pub fn create_watcher(&mut self, hash: H) -> watcher::Watcher> { let sender = self.watchers.entry(hash.clone()).or_insert_with(watcher::Sender::default); sender.new_watcher(hash) } @@ -101,8 +105,34 @@ impl Listene } /// Transaction was pruned from the pool. - pub fn pruned(&mut self, header_hash: H2, tx: &H) { - debug!(target: "txpool", "[{:?}] Pruned at {:?}", tx, header_hash); - self.fire(tx, |watcher| watcher.in_block(header_hash)) + pub fn pruned(&mut self, block_hash: BlockHash, tx: &H) { + debug!(target: "txpool", "[{:?}] Pruned at {:?}", tx, block_hash); + self.fire(tx, |s| s.in_block(block_hash)); + self.finality_watchers.entry(block_hash).or_insert(vec![]).push(tx.clone()); + + while self.finality_watchers.len() > MAX_FINALITY_WATCHERS { + if let Some((hash, txs)) = self.finality_watchers.pop_front() { + for tx in txs { + self.fire(&tx, |s| s.finality_timeout(hash.clone())); + } + } + } + } + + /// The block this transaction was included in has been retracted. + pub fn retracted(&mut self, block_hash: BlockHash) { + if let Some(hashes) = self.finality_watchers.remove(&block_hash) { + for hash in hashes { + self.fire(&hash, |s| s.retracted(block_hash)) + } + } + } + + /// Notify all watchers that transactions have been finalized + pub fn finalized(&mut self, block_hash: BlockHash, txs: Vec) { + self.finality_watchers.remove(&block_hash); + for h in txs { + self.fire(&h, |s| s.finalized(block_hash.clone())) + } } } diff --git a/client/transaction-pool/graph/src/pool.rs b/client/transaction-pool/graph/src/pool.rs index ab4e3a5a79..392abdca48 100644 --- a/client/transaction-pool/graph/src/pool.rs +++ b/client/transaction-pool/graph/src/pool.rs @@ -33,7 +33,8 @@ use sp_runtime::{ traits::{self, SaturatedConversion}, transaction_validity::{TransactionValidity, TransactionTag as Tag, TransactionValidityError}, }; -use sp_transaction_pool::{error, PoolStatus}; +use sp_transaction_pool::error; +use wasm_timer::Instant; use crate::validated_pool::{ValidatedPool, ValidatedTransaction}; @@ -122,6 +123,7 @@ pub struct Pool { validated_pool: Arc>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for Pool where B::Hash: parity_util_mem::MallocSizeOf, @@ -189,7 +191,6 @@ impl Pool { at: &BlockId, max: Option, ) -> Result<(), B::Error> { - use std::time::Instant; log::debug!(target: "txpool", "Fetching ready transactions (up to: {})", max.map(|x| format!("{}", x)).unwrap_or_else(|| "all".into()) @@ -317,7 +318,7 @@ impl Pool { // Make sure that we don't revalidate extrinsics that were part of the recently // imported block. This is especially important for UTXO-like chains cause the // inputs are pruned so such transaction would go to future again. - self.validated_pool.ban(&std::time::Instant::now(), known_imported_hashes.clone().into_iter()); + self.validated_pool.ban(&Instant::now(), known_imported_hashes.clone().into_iter()); // Try to re-validate pruned transactions since some of them might be still valid. // note that `known_imported_hashes` will be rejected here due to temporary ban. @@ -337,34 +338,6 @@ impl Pool { ) } - /// Return an event stream of notifications for when transactions are imported to the pool. - /// - /// Consumers of this stream should use the `ready` method to actually get the - /// pending transactions in the right order. - pub fn import_notification_stream(&self) -> EventStream> { - self.validated_pool.import_notification_stream() - } - - /// Invoked when extrinsics are broadcasted. - pub fn on_broadcasted(&self, propagated: HashMap, Vec>) { - self.validated_pool.on_broadcasted(propagated) - } - - /// Remove invalid transactions from the pool. - pub fn remove_invalid(&self, hashes: &[ExHash]) -> Vec> { - self.validated_pool.remove_invalid(hashes) - } - - /// Get an iterator for ready transactions ordered by priority - pub fn ready(&self) -> impl Iterator> { - self.validated_pool.ready() - } - - /// Returns pool status. - pub fn status(&self) -> PoolStatus { - self.validated_pool.status() - } - /// Returns transaction hash pub fn hash_of(&self, xt: &ExtrinsicFor) -> ExHash { self.validated_pool.api().hash_and_length(xt).0 @@ -453,9 +426,9 @@ impl Pool { (hash, validity) } - /// Get ready transaction by hash, if it present in the pool. - pub fn ready_transaction(&self, hash: &ExHash) -> Option> { - self.validated_pool.ready_by_hash(hash) + /// get a reference to the underlying validated pool. + pub fn validated_pool(&self) -> &ValidatedPool { + &self.validated_pool } } @@ -469,10 +442,7 @@ impl Clone for Pool { #[cfg(test)] mod tests { - use std::{ - collections::{HashMap, HashSet}, - time::Instant, - }; + use std::collections::{HashMap, HashSet}; use parking_lot::Mutex; use futures::executor::block_on; use super::*; @@ -481,6 +451,7 @@ mod tests { use codec::Encode; use substrate_test_runtime::{Block, Extrinsic, Transfer, H256, AccountId}; use assert_matches::assert_matches; + use wasm_timer::Instant; use crate::base_pool::Limit; const INVALID_NONCE: u64 = 254; @@ -599,7 +570,7 @@ mod tests { }))).unwrap(); // then - assert_eq!(pool.ready().map(|v| v.hash).collect::>(), vec![hash]); + assert_eq!(pool.validated_pool().ready().map(|v| v.hash).collect::>(), vec![hash]); } #[test] @@ -616,8 +587,8 @@ mod tests { // when pool.validated_pool.rotator().ban(&Instant::now(), vec![pool.hash_of(&uxt)]); let res = block_on(pool.submit_one(&BlockId::Number(0), uxt)); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then assert_matches!(res.unwrap_err(), error::Error::TemporarilyBanned); @@ -628,7 +599,7 @@ mod tests { let stream = { // given let pool = pool(); - let stream = pool.import_notification_stream(); + let stream = pool.validated_pool().import_notification_stream(); // when let _hash = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -651,8 +622,8 @@ mod tests { nonce: 3, }))).unwrap(); - assert_eq!(pool.status().ready, 2); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().ready, 2); + assert_eq!(pool.validated_pool().status().future, 1); stream }; @@ -690,9 +661,9 @@ mod tests { pool.validated_pool.clear_stale(&BlockId::Number(5)).unwrap(); // then - assert_eq!(pool.ready().count(), 0); - assert_eq!(pool.status().future, 0); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().ready().count(), 0); + assert_eq!(pool.validated_pool().status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // make sure they are temporarily banned as well assert!(pool.validated_pool.rotator().is_banned(&hash1)); assert!(pool.validated_pool.rotator().is_banned(&hash2)); @@ -736,7 +707,7 @@ mod tests { amount: 5, nonce: 1, }))).unwrap(); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); // when let hash2 = block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -747,7 +718,7 @@ mod tests { }))).unwrap(); // then - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); assert!(pool.validated_pool.rotator().is_banned(&hash1)); assert!(!pool.validated_pool.rotator().is_banned(&hash2)); } @@ -774,8 +745,8 @@ mod tests { }))).unwrap_err(); // then - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); } #[test] @@ -792,8 +763,8 @@ mod tests { }))).unwrap_err(); // then - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); assert_matches!(err, error::Error::NoTagsProvided); } @@ -810,19 +781,18 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); // when block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![])).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); - assert_eq!(stream.next(), None); } #[test] @@ -835,19 +805,18 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); // when block_on(pool.prune_tags(&BlockId::Number(2), vec![vec![0u8]], vec![2u64])).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 0); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); assert_eq!(stream.next(), Some(TransactionStatus::Ready)); assert_eq!(stream.next(), Some(TransactionStatus::InBlock(H256::from_low_u64_be(2).into()))); - assert_eq!(stream.next(), None); } #[test] @@ -860,8 +829,8 @@ mod tests { amount: 5, nonce: 1, }))).unwrap(); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 1); // when block_on(pool.submit_one(&BlockId::Number(0), uxt(Transfer { @@ -870,7 +839,7 @@ mod tests { amount: 5, nonce: 0, }))).unwrap(); - assert_eq!(pool.status().ready, 2); + assert_eq!(pool.validated_pool().status().ready, 2); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); @@ -889,7 +858,7 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when pool.validated_pool.remove_invalid(&[*watcher.hash()]); @@ -913,13 +882,13 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), uxt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when let mut map = HashMap::new(); let peers = vec!["a".into(), "b".into(), "c".into()]; map.insert(*watcher.hash(), peers.clone()); - pool.on_broadcasted(map); + pool.validated_pool().on_broadcasted(map); // then @@ -948,7 +917,7 @@ mod tests { nonce: 0, }); let watcher = block_on(pool.submit_and_watch(&BlockId::Number(0), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // when let xt = uxt(Transfer { @@ -958,7 +927,7 @@ mod tests { nonce: 1, }); block_on(pool.submit_one(&BlockId::Number(1), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // then let mut stream = futures::executor::block_on_stream(watcher.into_stream()); @@ -1001,11 +970,11 @@ mod tests { // The tag the above transaction provides (TestApi is using just nonce as u8) let provides = vec![0_u8]; block_on(pool.submit_one(&BlockId::Number(0), xt)).unwrap(); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // Now block import happens before the second transaction is able to finish verification. block_on(pool.prune_tags(&BlockId::Number(1), vec![provides], vec![])).unwrap(); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // so when we release the verification of the previous one it will have @@ -1015,8 +984,8 @@ mod tests { // then is_ready.recv().unwrap(); // wait for finish - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 0); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 0); } } @@ -1048,7 +1017,7 @@ mod tests { let tx4 = transfer(4); let hash4 = pool.validated_pool.api().hash_and_length(&tx4).0; let watcher4 = block_on(pool.submit_and_watch(&BlockId::Number(0), tx4)).unwrap(); - assert_eq!(pool.status().ready, 5); + assert_eq!(pool.validated_pool().status().ready, 5); // when pool.validated_pool.api().invalidate.lock().insert(hash3); @@ -1065,7 +1034,7 @@ mod tests { // // events for hash3 are: Ready, Invalid // events for hash4 are: Ready, Invalid - assert_eq!(pool.status().ready, 2); + assert_eq!(pool.validated_pool().status().ready, 2); assert_eq!( futures::executor::block_on_stream(watcher3.into_stream()).collect::>(), vec![TransactionStatus::Ready, TransactionStatus::Invalid], @@ -1096,4 +1065,3 @@ mod tests { ); } } - diff --git a/client/transaction-pool/graph/src/rotator.rs b/client/transaction-pool/graph/src/rotator.rs index e1c852f95a..55a9230522 100644 --- a/client/transaction-pool/graph/src/rotator.rs +++ b/client/transaction-pool/graph/src/rotator.rs @@ -23,9 +23,10 @@ use std::{ collections::HashMap, hash, iter, - time::{Duration, Instant}, + time::Duration, }; use parking_lot::RwLock; +use wasm_timer::Instant; use crate::base_pool::Transaction; diff --git a/client/transaction-pool/graph/src/validated_pool.rs b/client/transaction-pool/graph/src/validated_pool.rs index 9524284064..8752414637 100644 --- a/client/transaction-pool/graph/src/validated_pool.rs +++ b/client/transaction-pool/graph/src/validated_pool.rs @@ -16,13 +16,11 @@ use std::{ collections::{HashSet, HashMap}, - fmt, hash, sync::Arc, - time, }; -use crate::base_pool as base; +use crate::{base_pool as base, BlockHash}; use crate::listener::Listener; use crate::rotator::PoolRotator; use crate::watcher::Watcher; @@ -37,9 +35,10 @@ use sp_runtime::{ transaction_validity::TransactionTag as Tag, }; use sp_transaction_pool::{error, PoolStatus}; +use wasm_timer::Instant; use crate::base_pool::PruneStatus; -use crate::pool::{EventStream, Options, ChainApi, BlockHash, ExHash, ExtrinsicFor, TransactionFor}; +use crate::pool::{EventStream, Options, ChainApi, ExHash, ExtrinsicFor, TransactionFor}; /// Pre-validated transaction. Validated pool only accepts transactions wrapped in this enum. #[derive(Debug)] @@ -62,10 +61,10 @@ pub type ValidatedTransactionFor = ValidatedTransaction< >; /// Pool that deals with validated transactions. -pub(crate) struct ValidatedPool { +pub struct ValidatedPool { api: Arc, options: Options, - listener: RwLock, BlockHash>>, + listener: RwLock, B>>, pool: RwLock, ExtrinsicFor, @@ -74,6 +73,7 @@ pub(crate) struct ValidatedPool { rotator: PoolRotator>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for ValidatedPool where B::Hash: parity_util_mem::MallocSizeOf, @@ -90,9 +90,9 @@ impl ValidatedPool { pub fn new(options: Options, api: Arc) -> Self { let base_pool = base::BasePool::new(options.reject_future_transactions); ValidatedPool { - api, options, listener: Default::default(), + api, pool: RwLock::new(base_pool), import_notification_sinks: Default::default(), rotator: Default::default(), @@ -100,7 +100,7 @@ impl ValidatedPool { } /// Bans given set of hashes. - pub fn ban(&self, now: &std::time::Instant, hashes: impl IntoIterator>) { + pub fn ban(&self, now: &Instant, hashes: impl IntoIterator>) { self.rotator.ban(now, hashes) } @@ -137,21 +137,22 @@ impl ValidatedPool { let imported = self.pool.write().import(tx)?; if let base::Imported::Ready { ref hash, .. } = imported { - self.import_notification_sinks.lock().retain(|sink| sink.unbounded_send(hash.clone()).is_ok()); + self.import_notification_sinks.lock() + .retain(|sink| sink.unbounded_send(hash.clone()).is_ok()); } let mut listener = self.listener.write(); fire_events(&mut *listener, &imported); Ok(imported.hash().clone()) - } + }, ValidatedTransaction::Invalid(hash, err) => { - self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash)); + self.rotator.ban(&Instant::now(), std::iter::once(hash)); Err(err.into()) }, ValidatedTransaction::Unknown(hash, err) => { self.listener.write().invalid(&hash, false); Err(err.into()) - } + }, } } @@ -177,7 +178,7 @@ impl ValidatedPool { let removed = pool.enforce_limits(ready_limit, future_limit) .into_iter().map(|x| x.hash.clone()).collect::>(); // ban all removed transactions - self.rotator.ban(&std::time::Instant::now(), removed.iter().map(|x| x.clone())); + self.rotator.ban(&Instant::now(), removed.iter().map(|x| x.clone())); removed }; // run notifications @@ -208,7 +209,7 @@ impl ValidatedPool { .map(|_| watcher) }, ValidatedTransaction::Invalid(hash, err) => { - self.rotator.ban(&std::time::Instant::now(), std::iter::once(hash)); + self.rotator.ban(&Instant::now(), std::iter::once(hash)); Err(err.into()) }, ValidatedTransaction::Unknown(_, err) => Err(err.into()), @@ -342,8 +343,7 @@ impl ValidatedPool { self.pool.read().by_hashes(&hashes) .into_iter() .map(|existing_in_pool| existing_in_pool - .map(|transaction| transaction.provides.iter().cloned() - .collect())) + .map(|transaction| transaction.provides.iter().cloned().collect())) .collect() } @@ -415,8 +415,14 @@ impl ValidatedPool { let header_hash = self.api.block_id_to_hash(at)? .ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())?; let mut listener = self.listener.write(); + let mut set = HashSet::with_capacity(hashes.size_hint().0); for h in hashes { - listener.pruned(header_hash, &h); + // `hashes` has possibly duplicate hashes. + // we'd like to send out the `InBlock` notification only once. + if !set.contains(&h) { + listener.pruned(header_hash, &h); + set.insert(h); + } } Ok(()) } @@ -430,7 +436,7 @@ impl ValidatedPool { let block_number = self.api.block_id_to_number(at)? .ok_or_else(|| error::Error::InvalidBlockId(format!("{:?}", at)).into())? .saturated_into::(); - let now = time::Instant::now(); + let now = Instant::now(); let to_remove = { self.ready() .filter(|tx| self.rotator.ban_if_stale(&now, block_number, &tx)) @@ -467,7 +473,10 @@ impl ValidatedPool { &self.api } - /// Return an event stream of transactions imported to the pool. + /// Return an event stream of notifications for when transactions are imported to the pool. + /// + /// Consumers of this stream should use the `ready` method to actually get the + /// pending transactions in the right order. pub fn import_notification_stream(&self) -> EventStream> { let (sink, stream) = mpsc::unbounded(); self.import_notification_sinks.lock().push(sink); @@ -491,13 +500,13 @@ impl ValidatedPool { pub fn remove_invalid(&self, hashes: &[ExHash]) -> Vec> { // early exit in case there is no invalid transactions. if hashes.is_empty() { - return vec![] + return vec![]; } debug!(target: "txpool", "Removing invalid transactions: {:?}", hashes); // temporarily ban invalid transactions - self.rotator.ban(&time::Instant::now(), hashes.iter().cloned()); + self.rotator.ban(&Instant::now(), hashes.iter().cloned()); let invalid = self.pool.write().remove_subtree(hashes); @@ -520,14 +529,34 @@ impl ValidatedPool { pub fn status(&self) -> PoolStatus { self.pool.read().status() } + + /// Notify all watchers that transactions in the block with hash have been finalized + pub async fn on_block_finalized(&self, block_hash: BlockHash) -> Result<(), B::Error> { + debug!(target: "txpool", "Attempting to notify watchers of finalization for {}", block_hash); + // fetch all extrinsic hashes + if let Some(txs) = self.api.block_body(&BlockId::Hash(block_hash.clone())).await? { + let tx_hashes = txs.into_iter() + .map(|tx| self.api.hash_and_length(&tx).0) + .collect::>(); + // notify the watcher that these extrinsics have been finalized + self.listener.write().finalized(block_hash, tx_hashes); + } + + Ok(()) + } + + /// Notify the listener of retracted blocks + pub fn on_block_retracted(&self, block_hash: BlockHash) { + self.listener.write().retracted(block_hash) + } } -fn fire_events( - listener: &mut Listener, +fn fire_events( + listener: &mut Listener, imported: &base::Imported, ) where H: hash::Hash + Eq + traits::Member + Serialize, - H2: Clone + fmt::Debug, + B: ChainApi, { match *imported { base::Imported::Ready { ref promoted, ref failed, ref removed, ref hash } => { diff --git a/client/transaction-pool/graph/src/watcher.rs b/client/transaction-pool/graph/src/watcher.rs index f9c234f73c..d28f6814e4 100644 --- a/client/transaction-pool/graph/src/watcher.rs +++ b/client/transaction-pool/graph/src/watcher.rs @@ -26,12 +26,12 @@ use sp_transaction_pool::TransactionStatus; /// /// Represents a stream of status updates for particular extrinsic. #[derive(Debug)] -pub struct Watcher { - receiver: mpsc::UnboundedReceiver>, +pub struct Watcher { + receiver: mpsc::UnboundedReceiver>, hash: H, } -impl Watcher { +impl Watcher { /// Returns the transaction hash. pub fn hash(&self) -> &H { &self.hash @@ -40,30 +40,30 @@ impl Watcher { /// Pipe the notifications to given sink. /// /// Make sure to drive the future to completion. - pub fn into_stream(self) -> impl Stream> { + pub fn into_stream(self) -> impl Stream> { self.receiver } } /// Sender part of the watcher. Exposed only for testing purposes. #[derive(Debug)] -pub struct Sender { - receivers: Vec>>, - finalized: bool, +pub struct Sender { + receivers: Vec>>, + is_finalized: bool, } -impl Default for Sender { +impl Default for Sender { fn default() -> Self { Sender { receivers: Default::default(), - finalized: false, + is_finalized: false, } } } -impl Sender { +impl Sender { /// Add a new watcher to this sender object. - pub fn new_watcher(&mut self, hash: H) -> Watcher { + pub fn new_watcher(&mut self, hash: H) -> Watcher { let (tx, receiver) = mpsc::unbounded(); self.receivers.push(tx); Watcher { @@ -85,26 +85,42 @@ impl Sender { /// Some state change (perhaps another extrinsic was included) rendered this extrinsic invalid. pub fn usurped(&mut self, hash: H) { self.send(TransactionStatus::Usurped(hash)); - self.finalized = true; + self.is_finalized = true; } /// Extrinsic has been included in block with given hash. - pub fn in_block(&mut self, hash: H2) { + pub fn in_block(&mut self, hash: BH) { self.send(TransactionStatus::InBlock(hash)); - self.finalized = true; + } + + /// Extrinsic has been finalized by a finality gadget. + pub fn finalized(&mut self, hash: BH) { + self.send(TransactionStatus::Finalized(hash)); + self.is_finalized = true; + } + + /// The block this extrinsic was included in has been retracted + pub fn finality_timeout(&mut self, hash: BH) { + self.send(TransactionStatus::FinalityTimeout(hash)); + self.is_finalized = true; + } + + /// The block this extrinsic was included in has been retracted + pub fn retracted(&mut self, hash: BH) { + self.send(TransactionStatus::Retracted(hash)); } /// Extrinsic has been marked as invalid by the block builder. pub fn invalid(&mut self) { self.send(TransactionStatus::Invalid); // we mark as finalized as there are no more notifications - self.finalized = true; + self.is_finalized = true; } /// Transaction has been dropped from the pool because of the limit. pub fn dropped(&mut self) { self.send(TransactionStatus::Dropped); - self.finalized = true; + self.is_finalized = true; } /// The extrinsic has been broadcast to the given peers. @@ -114,10 +130,10 @@ impl Sender { /// Returns true if the are no more listeners for this extrinsic or it was finalized. pub fn is_done(&self) -> bool { - self.finalized || self.receivers.is_empty() + self.is_finalized || self.receivers.is_empty() } - fn send(&mut self, status: TransactionStatus) { + fn send(&mut self, status: TransactionStatus) { self.receivers.retain(|sender| sender.unbounded_send(status.clone()).is_ok()) } } diff --git a/client/transaction-pool/src/api.rs b/client/transaction-pool/src/api.rs index bfc13c01fd..84e06cc33e 100644 --- a/client/transaction-pool/src/api.rs +++ b/client/transaction-pool/src/api.rs @@ -64,7 +64,8 @@ impl FullChainApi where impl sc_transaction_graph::ChainApi for FullChainApi where Block: BlockT, - Client: ProvideRuntimeApi + BlockBody + BlockIdTo + 'static + Send + Sync, + Client: ProvideRuntimeApi + BlockBody + BlockIdTo, + Client: Send + Sync + 'static, Client::Api: TaggedTransactionQueue, sp_api::ApiErrorFor: Send, { diff --git a/client/transaction-pool/src/lib.rs b/client/transaction-pool/src/lib.rs index 7084e1c4a0..fbfc6a24e6 100644 --- a/client/transaction-pool/src/lib.rs +++ b/client/transaction-pool/src/lib.rs @@ -28,19 +28,19 @@ pub mod testing; pub use sc_transaction_graph as txpool; pub use crate::api::{FullChainApi, LightChainApi}; -use std::{collections::HashMap, sync::Arc, pin::Pin, time::Instant}; +use std::{collections::HashMap, sync::Arc, pin::Pin}; use futures::{Future, FutureExt, future::ready}; use parking_lot::Mutex; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, NumberFor, SimpleArithmetic, Extrinsic}, + traits::{Block as BlockT, NumberFor, AtLeast32Bit, Extrinsic}, }; use sp_transaction_pool::{ - TransactionPool, PoolStatus, ImportNotificationStream, - TxHash, TransactionFor, TransactionStatusStreamFor, BlockHash, - MaintainedTransactionPool, PoolFuture, + TransactionPool, PoolStatus, ImportNotificationStream, TxHash, TransactionFor, + TransactionStatusStreamFor, MaintainedTransactionPool, PoolFuture, ChainEvent, }; +use wasm_timer::Instant; /// Basic implementation of transaction pool that can be customized by providing PoolApi. pub struct BasicPool @@ -53,6 +53,7 @@ pub struct BasicPool revalidation_strategy: Arc>>>, } +#[cfg(not(target_os = "unknown"))] impl parity_util_mem::MallocSizeOf for BasicPool where PoolApi: sc_transaction_graph::ChainApi, @@ -113,7 +114,6 @@ impl BasicPool } )), } - } /// Gets shared reference to the underlying pool. @@ -172,19 +172,19 @@ impl TransactionPool for BasicPool } fn remove_invalid(&self, hashes: &[TxHash]) -> Vec> { - self.pool.remove_invalid(hashes) + self.pool.validated_pool().remove_invalid(hashes) } fn status(&self) -> PoolStatus { - self.pool.status() + self.pool.validated_pool().status() } fn ready(&self) -> Box>> { - Box::new(self.pool.ready()) + Box::new(self.pool.validated_pool().ready()) } fn import_notification_stream(&self) -> ImportNotificationStream> { - self.pool.import_notification_stream() + self.pool.validated_pool().import_notification_stream() } fn hash_of(&self, xt: &TransactionFor) -> TxHash { @@ -192,11 +192,11 @@ impl TransactionPool for BasicPool } fn on_broadcasted(&self, propagations: HashMap, Vec>) { - self.pool.on_broadcasted(propagations) + self.pool.validated_pool().on_broadcasted(propagations) } fn ready_transaction(&self, hash: &TxHash) -> Option> { - self.pool.ready_transaction(hash) + self.pool.validated_pool().ready_by_hash(hash) } } @@ -205,14 +205,14 @@ enum RevalidationStatus { /// The revalidation has never been completed. NotScheduled, /// The revalidation is scheduled. - Scheduled(Option, Option), + Scheduled(Option, Option), /// The revalidation is in progress. InProgress, } enum RevalidationStrategy { Always, - Light(RevalidationStatus) + Light(RevalidationStatus), } struct RevalidationAction { @@ -221,7 +221,7 @@ struct RevalidationAction { revalidate_amount: Option, } -impl RevalidationStrategy { +impl RevalidationStrategy { pub fn clear(&mut self) { if let Self::Light(status) = self { status.clear() @@ -239,7 +239,7 @@ impl RevalidationStrategy { revalidate: status.next_required( block, revalidate_time_period, - revalidate_block_period + revalidate_block_period, ), resubmit: false, revalidate_amount: None, @@ -253,7 +253,7 @@ impl RevalidationStrategy { } } -impl RevalidationStatus { +impl RevalidationStatus { /// Called when revalidation is completed. pub fn clear(&mut self) { *self = Self::NotScheduled; @@ -273,7 +273,7 @@ impl RevalidationStatus { revalidate_block_period.map(|period| block + period), ); false - }, + } Self::Scheduled(revalidate_at_time, revalidate_at_block) => { let is_required = revalidate_at_time.map(|at| Instant::now() >= at).unwrap_or(false) || revalidate_at_block.map(|at| block >= at).unwrap_or(false); @@ -281,87 +281,105 @@ impl RevalidationStatus { *self = Self::InProgress; } is_required - }, + } Self::InProgress => false, } } } impl MaintainedTransactionPool for BasicPool -where - Block: BlockT, - PoolApi: 'static + sc_transaction_graph::ChainApi, + where + Block: BlockT, + PoolApi: 'static + sc_transaction_graph::ChainApi, { - fn maintain(&self, id: &BlockId, retracted: &[BlockHash]) - -> Pin + Send>> - { - let id = id.clone(); - let pool = self.pool.clone(); - let api = self.api.clone(); - - let block_number = match api.block_id_to_number(&id) { - Ok(Some(number)) => number, - _ => { - log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id); - return Box::pin(ready(())); - } - }; - - let next_action = self.revalidation_strategy.lock().next( - block_number, - Some(std::time::Duration::from_secs(60)), - Some(20.into()), - ); - let revalidation_strategy = self.revalidation_strategy.clone(); - let retracted = retracted.to_vec(); - - async move { - // We don't query block if we won't prune anything - if !pool.status().is_empty() { - let hashes = api.block_body(&id).await - .unwrap_or_else(|e| { - log::warn!("Prune known transactions: error request {:?}!", e); - None - }) - .unwrap_or_default() - .into_iter() - .map(|tx| pool.hash_of(&tx)) - .collect::>(); - - if let Err(e) = pool.prune_known(&id, &hashes) { - log::error!("Cannot prune known in the pool {:?}!", e); - } - } - - if next_action.resubmit { - let mut resubmit_transactions = Vec::new(); - - for retracted_hash in retracted { - let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await - .unwrap_or_else(|e| { - log::warn!("Failed to fetch block body {:?}!", e); - None - }) - .unwrap_or_default() - .into_iter() - .filter(|tx| tx.is_signed().unwrap_or(true)); - - resubmit_transactions.extend(block_transactions); - } - if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await { - log::debug!(target: "txpool", - "[{:?}] Error re-submitting transactions: {:?}", id, e - ) - } + fn maintain(&self, event: ChainEvent) -> Pin + Send>> { + match event { + ChainEvent::NewBlock { id, retracted, .. } => { + let id = id.clone(); + let pool = self.pool.clone(); + let api = self.api.clone(); + + let block_number = match api.block_id_to_number(&id) { + Ok(Some(number)) => number, + _ => { + log::trace!(target: "txqueue", "Skipping chain event - no number for that block {:?}", id); + return Box::pin(ready(())); + } + }; + + let next_action = self.revalidation_strategy.lock().next( + block_number, + Some(std::time::Duration::from_secs(60)), + Some(20.into()), + ); + let revalidation_strategy = self.revalidation_strategy.clone(); + let retracted = retracted.clone(); + + async move { + // We don't query block if we won't prune anything + if !pool.validated_pool().status().is_empty() { + let hashes = api.block_body(&id).await + .unwrap_or_else(|e| { + log::warn!("Prune known transactions: error request {:?}!", e); + None + }) + .unwrap_or_default() + .into_iter() + .map(|tx| pool.hash_of(&tx)) + .collect::>(); + + if let Err(e) = pool.prune_known(&id, &hashes) { + log::error!("Cannot prune known in the pool {:?}!", e); + } + } + + if next_action.resubmit { + let mut resubmit_transactions = Vec::new(); + + for retracted_hash in retracted { + // notify txs awaiting finality that it has been retracted + pool.validated_pool().on_block_retracted(retracted_hash.clone()); + + let block_transactions = api.block_body(&BlockId::hash(retracted_hash.clone())).await + .unwrap_or_else(|e| { + log::warn!("Failed to fetch block body {:?}!", e); + None + }) + .unwrap_or_default() + .into_iter() + .filter(|tx| tx.is_signed().unwrap_or(true)); + + resubmit_transactions.extend(block_transactions); + } + if let Err(e) = pool.submit_at(&id, resubmit_transactions, true).await { + log::debug!( + target: "txpool", + "[{:?}] Error re-submitting transactions: {:?}", id, e + ) + } + } + + if next_action.revalidate { + if let Err(e) = pool.revalidate_ready(&id, next_action.revalidate_amount).await { + log::warn!("Revalidate ready failed {:?}", e); + } + } + + revalidation_strategy.lock().clear(); + }.boxed() } - - if next_action.revalidate { - if let Err(e) = pool.revalidate_ready(&id, next_action.revalidate_amount).await { - log::warn!("Revalidate ready failed {:?}", e); - } + ChainEvent::Finalized { hash } => { + let pool = self.pool.clone(); + async move { + if let Err(e) = pool.validated_pool().on_block_finalized(hash).await { + log::warn!( + target: "txpool", + "Error [{}] occurred while attempting to notify watchers of finalization {}", + e, hash + ) + } + }.boxed() } - - revalidation_strategy.lock().clear(); - }.boxed() + } } } diff --git a/client/transaction-pool/src/testing/pool.rs b/client/transaction-pool/src/testing/pool.rs index fed02067b1..4a4f4638df 100644 --- a/client/transaction-pool/src/testing/pool.rs +++ b/client/transaction-pool/src/testing/pool.rs @@ -15,17 +15,18 @@ // along with Substrate. If not, see . use crate::*; -use sc_transaction_graph::Pool; use futures::executor::block_on; +use txpool::{self, Pool}; use sp_runtime::{ generic::BlockId, transaction_validity::ValidTransaction, }; use substrate_test_runtime_client::{ - runtime::{Block, Hash, Index}, + runtime::{Block, Hash, Index, Header}, AccountKeyring::*, }; use substrate_test_runtime_transaction_pool::{TestApi, uxt}; +use sp_transaction_pool::TransactionStatus; fn pool() -> Pool { Pool::new(Default::default(), TestApi::with_alice_nonce(209).into()) @@ -35,12 +36,22 @@ fn maintained_pool() -> BasicPool { BasicPool::new(Default::default(), std::sync::Arc::new(TestApi::with_alice_nonce(209))) } +fn header(number: u64) -> Header { + Header { + number, + digest: Default::default(), + extrinsics_root: Default::default(), + parent_hash: Default::default(), + state_root: Default::default(), + } +} + #[test] fn submission_should_work() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209]); } @@ -50,7 +61,7 @@ fn multiple_submission_should_work() { block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); } @@ -59,7 +70,7 @@ fn early_nonce_should_be_culled() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 208))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); } @@ -68,11 +79,11 @@ fn late_nonce_should_be_queued() { let pool = pool(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); } @@ -82,7 +93,7 @@ fn prune_tags_should_work() { let hash209 = block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 209))).unwrap(); block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 210))).unwrap(); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![209, 210]); block_on( @@ -93,7 +104,7 @@ fn prune_tags_should_work() { ) ).expect("Prune tags"); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![210]); } @@ -102,11 +113,11 @@ fn should_ban_invalid_transactions() { let pool = pool(); let uxt = uxt(Alice, 209); let hash = block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap(); - pool.remove_invalid(&[hash]); + pool.validated_pool().remove_invalid(&[hash]); block_on(pool.submit_one(&BlockId::number(0), uxt.clone())).unwrap_err(); // when - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, Vec::::new()); // then @@ -122,29 +133,29 @@ fn should_correctly_prune_transactions_providing_more_than_one_tag() { let pool = Pool::new(Default::default(), api.clone()); let xt = uxt(Alice, 209); block_on(pool.submit_one(&BlockId::number(0), xt.clone())).expect("1. Imported"); - assert_eq!(pool.status().ready, 1); + assert_eq!(pool.validated_pool().status().ready, 1); // remove the transaction that just got imported. api.increment_nonce(Alice.into()); block_on(pool.prune_tags(&BlockId::number(1), vec![vec![209]], vec![])).expect("1. Pruned"); - assert_eq!(pool.status().ready, 0); + assert_eq!(pool.validated_pool().status().ready, 0); // it's re-imported to future - assert_eq!(pool.status().future, 1); + assert_eq!(pool.validated_pool().status().future, 1); // so now let's insert another transaction that also provides the 155 api.increment_nonce(Alice.into()); let xt = uxt(Alice, 211); block_on(pool.submit_one(&BlockId::number(2), xt.clone())).expect("2. Imported"); - assert_eq!(pool.status().ready, 1); - assert_eq!(pool.status().future, 1); - let pending: Vec<_> = pool.ready().map(|a| a.data.transfer().nonce).collect(); + assert_eq!(pool.validated_pool().status().ready, 1); + assert_eq!(pool.validated_pool().status().future, 1); + let pending: Vec<_> = pool.validated_pool().ready().map(|a| a.data.transfer().nonce).collect(); assert_eq!(pending, vec![211]); // prune it and make sure the pool is empty api.increment_nonce(Alice.into()); block_on(pool.prune_tags(&BlockId::number(3), vec![vec![155]], vec![])).expect("2. Pruned"); - assert_eq!(pool.status().ready, 0); - assert_eq!(pool.status().future, 2); + assert_eq!(pool.validated_pool().status().ready, 0); + assert_eq!(pool.validated_pool().status().future, 2); } #[test] @@ -158,7 +169,14 @@ fn should_prune_old_during_maintenance() { pool.api.push_block(1, vec![xt.clone()]); - block_on(pool.maintain(&BlockId::number(1), &[])); + let event = ChainEvent::NewBlock { + id: BlockId::number(1), + is_new_best: true, + retracted: vec![], + header: header(1), + }; + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); } @@ -174,8 +192,14 @@ fn should_revalidate_during_maintenance() { assert_eq!(pool.api.validation_requests().len(), 2); pool.api.push_block(1, vec![xt1.clone()]); - - block_on(pool.maintain(&BlockId::number(1), &[])); + let event = ChainEvent::NewBlock { + id: BlockId::number(1), + is_new_best: true, + retracted: vec![], + header: header(1), + }; + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 1); // test that pool revalidated transaction that left ready and not included in the block assert_eq!(pool.api.validation_requests().len(), 3); @@ -193,8 +217,14 @@ fn should_resubmit_from_retracted_during_maintaince() { pool.api.push_block(1, vec![]); pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); - - block_on(pool.maintain(&BlockId::number(1), &[retracted_hash])); + let event = ChainEvent::NewBlock { + id: BlockId::Number(1), + is_new_best: true, + header: header(1), + retracted: vec![retracted_hash] + }; + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 1); } @@ -212,7 +242,14 @@ fn should_not_retain_invalid_hashes_from_retracted() { pool.api.push_fork_block(retracted_hash, vec![xt.clone()]); pool.api.add_invalid(&xt); - block_on(pool.maintain(&BlockId::number(1), &[retracted_hash])); + let event = ChainEvent::NewBlock { + id: BlockId::Number(1), + is_new_best: true, + header: header(1), + retracted: vec![retracted_hash] + }; + + block_on(pool.maintain(event)); assert_eq!(pool.status().ready, 0); } @@ -225,4 +262,188 @@ fn can_track_heap_size() { block_on(pool.submit_one(&BlockId::number(0), uxt(Alice, 212))).expect("1. Imported"); assert!(parity_util_mem::malloc_size(&pool) > 3000); -} \ No newline at end of file +} + +#[test] +fn finalization() { + let xt = uxt(Alice, 209); + let api = TestApi::with_alice_nonce(209); + api.push_block(1, vec![]); + let pool = BasicPool::new(Default::default(), api.into()); + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + pool.api.push_block(2, vec![xt.clone()]); + + let header = pool.api.chain().read().header_by_number.get(&2).cloned().unwrap(); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + block_on(pool.maintain(event)); + + let event = ChainEvent::Finalized { hash: header.hash() }; + block_on(pool.maintain(event)); + + let mut stream = futures::executor::block_on_stream(watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(header.hash()))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(header.hash()))); + assert_eq!(stream.next(), None); +} + +#[test] +fn fork_aware_finalization() { + let api = TestApi::empty(); + // starting block A1 (last finalized.) + api.push_block(1, vec![]); + + let pool = BasicPool::new(Default::default(), api.into()); + let mut canon_watchers = vec![]; + + let from_alice = uxt(Alice, 1); + let from_dave = uxt(Dave, 1); + let from_bob = uxt(Bob, 1); + let from_charlie = uxt(Charlie, 1); + pool.api.increment_nonce(Alice.into()); + pool.api.increment_nonce(Dave.into()); + pool.api.increment_nonce(Charlie.into()); + pool.api.increment_nonce(Bob.into()); + + let from_dave_watcher; + let from_bob_watcher; + let b1; + let d1; + let c2; + let d2; + + + // block B1 + { + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_alice.clone())).expect("1. Imported"); + let header = pool.api.push_block(2, vec![from_alice.clone()]); + canon_watchers.push((watcher, header.hash())); + + let event = ChainEvent::NewBlock { + id: BlockId::Number(2), + is_new_best: true, + header: header.clone(), + retracted: vec![], + }; + b1 = header.hash(); + block_on(pool.maintain(event)); + let event = ChainEvent::Finalized { hash: b1 }; + block_on(pool.maintain(event)); + } + + // block C2 + { + let header = pool.api.push_fork_block_with_parent(b1, vec![from_dave.clone()]); + from_dave_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_dave.clone())) + .expect("1. Imported"); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + c2 = header.hash(); + block_on(pool.maintain(event)); + } + + // block D2 + { + from_bob_watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_bob.clone())).expect("1. Imported"); + let header = pool.api.push_fork_block_with_parent(c2, vec![from_bob.clone()]); + + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + d2 = header.hash(); + block_on(pool.maintain(event)); + } + + // block C1 + { + let watcher = block_on(pool.submit_and_watch(&BlockId::number(1), from_charlie.clone())).expect("1.Imported"); + let header = pool.api.push_block(3, vec![from_charlie.clone()]); + + canon_watchers.push((watcher, header.hash())); + let event = ChainEvent::NewBlock { + id: BlockId::Number(3), + is_new_best: true, + header: header.clone(), + retracted: vec![c2, d2], + }; + block_on(pool.maintain(event)); + let event = ChainEvent::Finalized { hash: header.hash() }; + block_on(pool.maintain(event)); + } + + // block D1 + { + let xt = uxt(Eve, 0); + let w = block_on(pool.submit_and_watch(&BlockId::number(1), xt.clone())).expect("1. Imported"); + let header = pool.api.push_block(4, vec![xt.clone()]); + canon_watchers.push((w, header.hash())); + + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + d1 = header.hash(); + block_on(pool.maintain(event)); + let event = ChainEvent::Finalized { hash: d1 }; + block_on(pool.maintain(event)); + } + + let e1; + + // block e1 + { + let header = pool.api.push_block(5, vec![from_dave]); + e1 = header.hash(); + let event = ChainEvent::NewBlock { + id: BlockId::Hash(header.hash()), + is_new_best: true, + header: header.clone(), + retracted: vec![] + }; + block_on(pool.maintain(event)); + block_on(pool.maintain(ChainEvent::Finalized { hash: e1 })); + } + + + for (canon_watcher, h) in canon_watchers { + let mut stream = futures::executor::block_on_stream(canon_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(h.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(h))); + assert_eq!(stream.next(), None); + } + + + { + let mut stream= futures::executor::block_on_stream(from_dave_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(c2.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Retracted(c2))); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(e1))); + assert_eq!(stream.next(), Some(TransactionStatus::Finalized(e1.clone()))); + assert_eq!(stream.next(), None); + } + + { + let mut stream= futures::executor::block_on_stream(from_bob_watcher); + assert_eq!(stream.next(), Some(TransactionStatus::Ready)); + assert_eq!(stream.next(), Some(TransactionStatus::InBlock(d2.clone()))); + assert_eq!(stream.next(), Some(TransactionStatus::Retracted(d2))); + } + +} diff --git a/frame/assets/src/lib.rs b/frame/assets/src/lib.rs index ba6cc26448..c722a445cf 100644 --- a/frame/assets/src/lib.rs +++ b/frame/assets/src/lib.rs @@ -133,7 +133,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use frame_support::{Parameter, decl_module, decl_event, decl_storage, decl_error, ensure}; -use sp_runtime::traits::{Member, SimpleArithmetic, Zero, StaticLookup}; +use sp_runtime::traits::{Member, AtLeast32Bit, Zero, StaticLookup}; use frame_system::{self as system, ensure_signed}; use sp_runtime::traits::One; @@ -143,10 +143,10 @@ pub trait Trait: frame_system::Trait { type Event: From> + Into<::Event>; /// The units in which we record balances. - type Balance: Member + Parameter + SimpleArithmetic + Default + Copy; + type Balance: Member + Parameter + AtLeast32Bit + Default + Copy; /// The arithmetic type of asset identifier. - type AssetId: Parameter + SimpleArithmetic + Default + Copy; + type AssetId: Parameter + AtLeast32Bit + Default + Copy; } decl_module! { diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index 1578d5c556..5921b1ba20 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -23,10 +23,10 @@ pub use pallet_timestamp; use sp_std::{result, prelude::*}; -use frame_support::{decl_storage, decl_module, traits::FindAuthor, traits::Get}; +use frame_support::{decl_storage, decl_module, traits::{FindAuthor, Get, Randomness as RandomnessT}}; use sp_timestamp::OnTimestampSet; -use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill}; -use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, RandomnessBeacon}; +use sp_runtime::{generic::DigestItem, ConsensusEngineId, Perbill, PerThing}; +use sp_runtime::traits::{IsMember, SaturatedConversion, Saturating, Hash}; use sp_staking::{ SessionIndex, offence::{Offence, Kind}, @@ -191,9 +191,13 @@ decl_module! { } } -impl RandomnessBeacon for Module { - fn random() -> [u8; VRF_OUTPUT_LENGTH] { - Self::randomness() +impl RandomnessT<::Hash> for Module { + fn random(subject: &[u8]) -> T::Hash { + let mut subject = subject.to_vec(); + subject.reserve(VRF_OUTPUT_LENGTH); + subject.extend_from_slice(&Self::randomness()[..]); + + ::Hashing::hash(&subject[..]) } } diff --git a/frame/babe/src/mock.rs b/frame/babe/src/mock.rs index 23f42fc1f9..210e3139d4 100644 --- a/frame/babe/src/mock.rs +++ b/frame/babe/src/mock.rs @@ -20,7 +20,7 @@ use super::{Trait, Module, GenesisConfig}; use sp_consensus_babe::AuthorityId; use sp_runtime::{ - traits::IdentityLookup, Perbill, testing::{Header, UintAuthorityId}, impl_opaque_keys, + traits::IdentityLookup, Perbill, PerThing, testing::{Header, UintAuthorityId}, impl_opaque_keys, }; use sp_version::RuntimeVersion; use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; diff --git a/frame/balances/src/benchmarking.rs b/frame/balances/src/benchmarking.rs new file mode 100644 index 0000000000..7871b374c9 --- /dev/null +++ b/frame/balances/src/benchmarking.rs @@ -0,0 +1,322 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Balances pallet benchmarking. + +use super::*; + +use frame_system::RawOrigin; +use sp_io::hashing::blake2_256; +use sp_runtime::{BenchmarkResults, BenchmarkParameter}; +use sp_runtime::traits::{Bounded, Benchmarking, BenchmarkingSetup, Dispatchable}; + +use crate::Module as Balances; + +// Support Functions +fn account(name: &'static str, index: u32) -> T::AccountId { + let entropy = (name, index).using_encoded(blake2_256); + T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() +} + +// Benchmark `transfer` extrinsic with the worst possible conditions: +// * Transfer will kill the sender account. +// * Transfer will create the recipient account. +struct Transfer; +impl BenchmarkingSetup, RawOrigin> for Transfer { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Existential Deposit Multiplier + (BenchmarkParameter::E, 2, 1000), + // User Seed + (BenchmarkParameter::U, 1, 1000), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Constants + let ed = T::ExistentialDeposit::get(); + + // Select an account + let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; + let user = account::("user", u); + let user_origin = RawOrigin::Signed(user.clone()); + + // Give some multiple of the existential deposit + creation fee + transfer fee + let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + let mut balance = ed.saturating_mul(e.into()); + balance += T::CreationFee::get(); + let _ = as Currency<_>>::make_free_balance_be(&user, balance); + + // Transfer `e - 1` existential deposits + 1 unit, which guarantees to create one account, and reap this user. + let recipient = account::("recipient", u); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient); + let transfer_amt = ed.saturating_mul((e - 1).into()) + 1.into(); + + // Return the `transfer` call + Ok((crate::Call::::transfer(recipient_lookup, transfer_amt), user_origin)) + } +} + +// Benchmark `transfer` with the best possible condition: +// * Both accounts exist and will continue to exist. +struct TransferBestCase; +impl BenchmarkingSetup, RawOrigin> for TransferBestCase { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Existential Deposit Multiplier + (BenchmarkParameter::E, 2, 1000), + // User Seed + (BenchmarkParameter::U, 1, 1000), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Constants + let ed = T::ExistentialDeposit::get(); + + // Select a sender + let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; + let user = account::("user", u); + let user_origin = RawOrigin::Signed(user.clone()); + + // Select a recipient + let recipient = account::("recipient", u); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); + + // Get the existential deposit multiplier + let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + + // Give the sender account max funds for transfer (their account will never reasonably be killed). + let _ = as Currency<_>>::make_free_balance_be(&user, T::Balance::max_value()); + + // Give the recipient account existential deposit (thus their account already exists). + let _ = as Currency<_>>::make_free_balance_be(&recipient, ed); + + // Transfer e * existential deposit. + let transfer_amt = ed.saturating_mul(e.into()); + + // Return the `transfer` call + Ok((crate::Call::::transfer(recipient_lookup, transfer_amt), user_origin)) + } +} + +// Benchmark `transfer_keep_alive` with the worst possible condition: +// * The recipient account is created. +struct TransferKeepAlive; +impl BenchmarkingSetup, RawOrigin> for TransferKeepAlive { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Existential Deposit Multiplier + (BenchmarkParameter::E, 2, 1000), + // User Seed + (BenchmarkParameter::U, 1, 1000), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Constants + let ed = T::ExistentialDeposit::get(); + + // Select a sender + let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; + let user = account::("user", u); + let user_origin = RawOrigin::Signed(user.clone()); + + // Select a recipient + let recipient = account::("recipient", u); + let recipient_lookup: ::Source = T::Lookup::unlookup(recipient.clone()); + + // Get the existential deposit multiplier + let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + + // Give the sender account max funds, thus a transfer will not kill account. + let _ = as Currency<_>>::make_free_balance_be(&user, T::Balance::max_value()); + + // Transfer e * existential deposit. + let transfer_amt = ed.saturating_mul(e.into()); + + // Return the `transfer_keep_alive` call + Ok((crate::Call::::transfer_keep_alive(recipient_lookup, transfer_amt), user_origin)) + } +} + +// Benchmark `set_balance` coming from ROOT account. This always creates an account. +struct SetBalance; +impl BenchmarkingSetup, RawOrigin> for SetBalance { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Existential Deposit Multiplier + (BenchmarkParameter::E, 2, 1000), + // User Seed + (BenchmarkParameter::U, 1, 1000), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Constants + let ed = T::ExistentialDeposit::get(); + + // Select a sender + let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; + let user = account::("user", u); + let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); + + // Get the existential deposit multiplier for free and reserved + let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + let balance_amt = ed.saturating_mul(e.into()); + + // Return the `set_balance` call + Ok((crate::Call::::set_balance(user_lookup, balance_amt, balance_amt), RawOrigin::Root)) + } +} + +// Benchmark `set_balance` coming from ROOT account. This always kills an account. +struct SetBalanceKilling; +impl BenchmarkingSetup, RawOrigin> for SetBalanceKilling { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Existential Deposit Multiplier + (BenchmarkParameter::E, 2, 1000), + // User Seed + (BenchmarkParameter::U, 1, 1000), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Constants + let ed = T::ExistentialDeposit::get(); + + // Select a sender + let u = components.iter().find(|&c| c.0 == BenchmarkParameter::U).unwrap().1; + let user = account::("user", u); + let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); + + // Get the existential deposit multiplier for free and reserved + let e = components.iter().find(|&c| c.0 == BenchmarkParameter::E).unwrap().1; + // Give the user some initial balance + let balance_amt = ed.saturating_mul(e.into()); + let _ = as Currency<_>>::make_free_balance_be(&user, balance_amt); + + // Return the `set_balance` call that will kill the account + Ok((crate::Call::::set_balance(user_lookup, 0.into(), 0.into()), RawOrigin::Root)) + } +} + +// The list of available benchmarks for this pallet. +enum SelectedBenchmark { + Transfer, + TransferBestCase, + TransferKeepAlive, + SetBalance, + SetBalanceKilling, +} + +// Allow us to select a benchmark from the list of available benchmarks. +impl BenchmarkingSetup, RawOrigin> for SelectedBenchmark { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + match self { + Self::Transfer => , RawOrigin>>::components(&Transfer), + Self::TransferBestCase => , RawOrigin>>::components(&TransferBestCase), + Self::TransferKeepAlive => , RawOrigin>>::components(&TransferKeepAlive), + Self::SetBalance => , RawOrigin>>::components(&SetBalance), + Self::SetBalanceKilling => , RawOrigin>>::components(&SetBalanceKilling), + } + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + match self { + Self::Transfer => , RawOrigin>>::instance(&Transfer, components), + Self::TransferBestCase => , RawOrigin>>::instance(&TransferBestCase, components), + Self::TransferKeepAlive => , RawOrigin>>::instance(&TransferKeepAlive, components), + Self::SetBalance => , RawOrigin>>::instance(&SetBalance, components), + Self::SetBalanceKilling => , RawOrigin>>::instance(&SetBalanceKilling, components), + } + } +} + +impl Benchmarking for Module { + fn run_benchmark(extrinsic: Vec, steps: u32, repeat: u32) -> Result, &'static str> { + // Map the input to the selected benchmark. + let selected_benchmark = match extrinsic.as_slice() { + b"transfer" => SelectedBenchmark::Transfer, + b"transfer_best_case" => SelectedBenchmark::TransferBestCase, + b"transfer_keep_alive" => SelectedBenchmark::TransferKeepAlive, + b"set_balance" => SelectedBenchmark::SetBalance, + b"set_balance_killing" => SelectedBenchmark::SetBalanceKilling, + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + sp_io::benchmarking::commit_db(); + sp_io::benchmarking::wipe_db(); + + let components = , RawOrigin>>::components(&selected_benchmark); + // results go here + let mut results: Vec = Vec::new(); + // Select the component we will be benchmarking. Each component will be benchmarked. + for (name, low, high) in components.iter() { + // Create up to `STEPS` steps for that component between high and low. + let step_size = ((high - low) / steps).max(1); + let num_of_steps = (high - low) / step_size; + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = low + step_size * s; + + // Select the mid value for all the other components. + let c: Vec<(BenchmarkParameter, u32)> = components.iter() + .map(|(n, l, h)| + (*n, if n == name { component_value } else { (h - l) / 2 + l }) + ).collect(); + + // Run the benchmark `repeat` times. + for _r in 0..repeat { + // Set up the externalities environment for the setup we want to benchmark. + let (call, caller) = , RawOrigin>>::instance(&selected_benchmark, &c)?; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + sp_io::benchmarking::commit_db(); + // Run the benchmark. + let start = sp_io::benchmarking::current_time(); + call.dispatch(caller.clone().into())?; + let finish = sp_io::benchmarking::current_time(); + let elapsed = finish - start; + sp_std::if_std!{ + if let RawOrigin::Signed(who) = caller.clone() { + let balance = Account::::get(&who).free; + println!("Free Balance {:?}", balance); + } + } + results.push((c.clone(), elapsed)); + // Wipe the DB back to the genesis state. + sp_io::benchmarking::wipe_db(); + } + } + } + return Ok(results); + } +} diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 93968d61f5..d294847437 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -153,6 +153,7 @@ mod mock; #[cfg(test)] mod tests; mod migration; +mod benchmarking; use sp_std::prelude::*; use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr}; @@ -169,7 +170,7 @@ use frame_support::{ use sp_runtime::{ RuntimeDebug, DispatchResult, DispatchError, traits::{ - Zero, SimpleArithmetic, StaticLookup, Member, CheckedAdd, CheckedSub, + Zero, AtLeast32Bit, StaticLookup, Member, CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Saturating, Bounded, }, }; @@ -180,7 +181,7 @@ pub use self::imbalances::{PositiveImbalance, NegativeImbalance}; pub trait Subtrait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + SimpleArithmetic + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; /// A function that is invoked when the free-balance and the reserved-balance has fallen below @@ -202,7 +203,7 @@ pub trait Subtrait: frame_system::Trait { pub trait Trait: frame_system::Trait { /// The balance of an account. - type Balance: Parameter + Member + SimpleArithmetic + Codec + Default + Copy + + type Balance: Parameter + Member + AtLeast32Bit + Codec + Default + Copy + MaybeSerializeDeserialize + Debug; /// A function that is invoked when the free-balance and the reserved-balance has fallen below @@ -395,6 +396,10 @@ decl_storage! { config(balances): Vec<(T::AccountId, T::Balance)>; // ^^ begin, length, amount liquid at genesis build(|config: &GenesisConfig| { + assert!( + >::ExistentialDeposit::get() > Zero::zero(), + "The existential deposit should be greater than zero." + ); for (_, balance) in &config.balances { assert!( *balance >= >::ExistentialDeposit::get(), diff --git a/frame/balances/src/mock.rs b/frame/balances/src/mock.rs index 5c7278859d..ee8eccc02c 100644 --- a/frame/balances/src/mock.rs +++ b/frame/balances/src/mock.rs @@ -105,7 +105,7 @@ pub struct ExtBuilder { impl Default for ExtBuilder { fn default() -> Self { Self { - existential_deposit: 0, + existential_deposit: 1, creation_fee: 0, monied: false, } @@ -122,9 +122,6 @@ impl ExtBuilder { } pub fn monied(mut self, monied: bool) -> Self { self.monied = monied; - if self.existential_deposit == 0 { - self.existential_deposit = 1; - } self } pub fn set_associated_consts(&self) { diff --git a/frame/contracts/src/exec.rs b/frame/contracts/src/exec.rs index 610b030da8..547607dd85 100644 --- a/frame/contracts/src/exec.rs +++ b/frame/contracts/src/exec.rs @@ -1076,7 +1076,7 @@ mod tests { let mut gas_meter = GasMeter::::with_limit(1000, 1); - let result = ctx.instantiate(0, &mut gas_meter, &code, vec![]); + let result = ctx.instantiate(1, &mut gas_meter, &code, vec![]); assert_matches!(result, Ok(_)); let mut toks = gas_meter.tokens().iter(); @@ -1369,8 +1369,10 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader, None); + ctx.overlay.set_balance(&ALICE, 1); + let result = ctx.instantiate( - 0, + 1, &mut GasMeter::::with_limit(10000, 1), &input_data_ch, vec![1, 2, 3, 4], @@ -1415,6 +1417,7 @@ mod tests { ExtBuilder::default().build().execute_with(|| { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader, None); + ctx.overlay.set_balance(&BOB, 1); ctx.overlay.instantiate_contract(&BOB, recurse_ch).unwrap(); let result = ctx.call( @@ -1728,8 +1731,10 @@ mod tests { let cfg = Config::preload(); let mut ctx = ExecutionContext::top_level(ALICE, &cfg, &vm, &loader, None); + ctx.overlay.set_balance(&ALICE, 1); + let result = ctx.instantiate( - 0, + 1, &mut GasMeter::::with_limit(10000, 1), &rent_allowance_ch, vec![], diff --git a/frame/contracts/src/gas.rs b/frame/contracts/src/gas.rs index 6816bd3bfe..a65386d396 100644 --- a/frame/contracts/src/gas.rs +++ b/frame/contracts/src/gas.rs @@ -17,7 +17,7 @@ use crate::{GasSpent, Module, Trait, BalanceOf}; use sp_std::convert::TryFrom; use sp_runtime::traits::{ - CheckedMul, Zero, SaturatedConversion, SimpleArithmetic, UniqueSaturatedInto, + CheckedMul, Zero, SaturatedConversion, AtLeast32Bit, UniqueSaturatedInto, }; use frame_support::{ traits::{Currency, ExistenceRequirement, OnUnbalanced, WithdrawReason}, StorageValue, @@ -273,7 +273,7 @@ pub fn refund_unused_gas( /// A little handy utility for converting a value in balance units into approximate value in gas units /// at the given gas price. pub fn approx_gas_for_balance(gas_price: Balance, balance: Balance) -> Gas - where Balance: SimpleArithmetic + where Balance: AtLeast32Bit { (balance / gas_price).saturated_into::() } diff --git a/frame/contracts/src/tests.rs b/frame/contracts/src/tests.rs index 82801e1bd0..d5bc3027ff 100644 --- a/frame/contracts/src/tests.rs +++ b/frame/contracts/src/tests.rs @@ -238,7 +238,7 @@ pub struct ExtBuilder { impl Default for ExtBuilder { fn default() -> Self { Self { - existential_deposit: 0, + existential_deposit: 1, gas_price: 2, block_gas_limit: 100_000_000, transfer_fee: 0, @@ -701,6 +701,51 @@ fn dispatch_call_not_dispatched_after_top_level_transaction_failure() { }); } +const CODE_RUN_OUT_OF_GAS: &str = r#" +(module + (func (export "call") + (loop $inf (br $inf)) ;; just run out of gas + (unreachable) + ) + (func (export "deploy")) +) +"#; + +#[test] +fn run_out_of_gas() { + let (wasm, code_hash) = compile_module::(CODE_RUN_OUT_OF_GAS).unwrap(); + + ExtBuilder::default() + .existential_deposit(50) + .build() + .execute_with(|| { + Balances::deposit_creating(&ALICE, 1_000_000); + + assert_ok!(Contract::put_code(Origin::signed(ALICE), 100_000, wasm)); + + assert_ok!(Contract::instantiate( + Origin::signed(ALICE), + 100, + 100_000, + code_hash.into(), + vec![], + )); + + // Call the contract with a fixed gas limit. It must run out of gas because it just + // loops forever. + assert_err!( + Contract::call( + Origin::signed(ALICE), + BOB, // newly created account + 0, + 1000, + vec![], + ), + "ran out of gas during contract execution" + ); + }); +} + const CODE_SET_RENT: &str = r#" (module (import "env" "ext_dispatch_call" (func $ext_dispatch_call (param i32 i32))) diff --git a/frame/contracts/src/wasm/runtime.rs b/frame/contracts/src/wasm/runtime.rs index 5352b6cbe1..2e0c168d3e 100644 --- a/frame/contracts/src/wasm/runtime.rs +++ b/frame/contracts/src/wasm/runtime.rs @@ -42,6 +42,8 @@ const TRAP_RETURN_CODE: u32 = 0x0100; enum SpecialTrap { /// Signals that trap was generated in response to call `ext_return` host function. Return(Vec), + /// Signals that trap was generated because the contract exhausted its gas limit. + OutOfGas, } /// Can only be used for one call. @@ -77,9 +79,21 @@ pub(crate) fn to_execution_result( runtime: Runtime, sandbox_result: Result, ) -> ExecResult { - // Special case. The trap was the result of the execution `return` host function. - if let Some(SpecialTrap::Return(data)) = runtime.special_trap { - return Ok(ExecReturnValue { status: STATUS_SUCCESS, data }); + match runtime.special_trap { + // The trap was the result of the execution `return` host function. + Some(SpecialTrap::Return(data)) => { + return Ok(ExecReturnValue { + status: STATUS_SUCCESS, + data, + }) + } + Some(SpecialTrap::OutOfGas) => { + return Err(ExecError { + reason: "ran out of gas during contract execution".into(), + buffer: runtime.scratch_buf, + }) + } + _ => (), } // Check the exact type of the error. @@ -182,11 +196,15 @@ impl Token for RuntimeToken { fn charge_gas>( gas_meter: &mut GasMeter, metadata: &Tok::Metadata, + special_trap: &mut Option, token: Tok, ) -> Result<(), sp_sandbox::HostError> { match gas_meter.charge(metadata, token) { GasMeterResult::Proceed => Ok(()), - GasMeterResult::OutOfGas => Err(sp_sandbox::HostError), + GasMeterResult::OutOfGas => { + *special_trap = Some(SpecialTrap::OutOfGas); + Err(sp_sandbox::HostError) + }, } } @@ -203,7 +221,12 @@ fn read_sandbox_memory( ptr: u32, len: u32, ) -> Result, sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(len), + )?; let mut buf = vec![0u8; len as usize]; ctx.memory.get(ptr, buf.as_mut_slice()).map_err(|_| sp_sandbox::HostError)?; @@ -223,7 +246,12 @@ fn read_sandbox_memory_into_scratch( ptr: u32, len: u32, ) -> Result<(), sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(len), + )?; ctx.scratch_buf.resize(len as usize, 0); ctx.memory.get(ptr, ctx.scratch_buf.as_mut_slice()).map_err(|_| sp_sandbox::HostError)?; @@ -243,7 +271,12 @@ fn read_sandbox_memory_into_buf( ptr: u32, buf: &mut [u8], ) -> Result<(), sp_sandbox::HostError> { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReadMemory(buf.len() as u32))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReadMemory(buf.len() as u32), + )?; ctx.memory.get(ptr, buf).map_err(Into::into) } @@ -276,12 +309,18 @@ fn read_sandbox_memory_as( /// - designated area is not within the bounds of the sandbox memory. fn write_sandbox_memory( schedule: &Schedule, + special_trap: &mut Option, gas_meter: &mut GasMeter, memory: &sp_sandbox::Memory, ptr: u32, buf: &[u8], ) -> Result<(), sp_sandbox::HostError> { - charge_gas(gas_meter, schedule, RuntimeToken::WriteMemory(buf.len() as u32))?; + charge_gas( + gas_meter, + schedule, + special_trap, + RuntimeToken::WriteMemory(buf.len() as u32), + )?; memory.set(ptr, buf)?; @@ -303,7 +342,12 @@ define_env!(Env, , // // - amount: How much gas is used. gas(ctx, amount: u32) => { - charge_gas(&mut ctx.gas_meter, ctx.schedule, RuntimeToken::Explicit(amount))?; + charge_gas( + &mut ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::Explicit(amount) + )?; Ok(()) }, @@ -531,7 +575,12 @@ define_env!(Env, , // // This is the only way to return a data buffer to the caller. ext_return(ctx, data_ptr: u32, data_len: u32) => { - charge_gas(ctx.gas_meter, ctx.schedule, RuntimeToken::ReturnData(data_len))?; + charge_gas( + ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ReturnData(data_len) + )?; read_sandbox_memory_into_scratch(ctx, data_ptr, data_len)?; let output_buf = mem::replace(&mut ctx.scratch_buf, Vec::new()); @@ -670,7 +719,12 @@ define_env!(Env, , let balance_fee = <::T as Trait>::ComputeDispatchFee::compute_dispatch_fee(&call); approx_gas_for_balance(ctx.gas_meter.gas_price(), balance_fee) }; - charge_gas(&mut ctx.gas_meter, ctx.schedule, RuntimeToken::ComputedDispatchFee(fee))?; + charge_gas( + &mut ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ComputedDispatchFee(fee) + )?; ctx.ext.note_dispatch_call(call); @@ -688,7 +742,12 @@ define_env!(Env, , let balance_fee = <::T as Trait>::ComputeDispatchFee::compute_dispatch_fee(&call); approx_gas_for_balance(ctx.gas_meter.gas_price(), balance_fee) }; - charge_gas(&mut ctx.gas_meter, ctx.schedule, RuntimeToken::ComputedDispatchFee(fee))?; + charge_gas( + &mut ctx.gas_meter, + ctx.schedule, + &mut ctx.special_trap, + RuntimeToken::ComputedDispatchFee(fee) + )?; if let Some(doughnut) = ctx.ext.doughnut() { // Use of intermediate variable to avoid `#[warn(mutable_borrow_reservation_conflict)]` @@ -798,6 +857,7 @@ define_env!(Env, , // Finally, perform the write. write_sandbox_memory( ctx.schedule, + &mut ctx.special_trap, ctx.gas_meter, &ctx.memory, dest_ptr, @@ -845,6 +905,7 @@ define_env!(Env, , charge_gas( ctx.gas_meter, ctx.schedule, + &mut ctx.special_trap, RuntimeToken::DepositEvent(topics.len() as u32, data_len) )?; ctx.ext.deposit_event(topics, event_data); diff --git a/frame/democracy/src/lib.rs b/frame/democracy/src/lib.rs index ad24b34a62..80f6243f58 100644 --- a/frame/democracy/src/lib.rs +++ b/frame/democracy/src/lib.rs @@ -1231,7 +1231,7 @@ mod tests { type DelegatedDispatchVerifier = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index f34c25b044..8566cc5385 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -83,7 +83,10 @@ #![cfg_attr(not(feature = "std"), no_std)] use sp_std::prelude::*; -use sp_runtime::{print, DispatchResult, DispatchError, traits::{Zero, StaticLookup, Convert}}; +use sp_runtime::{ + print, DispatchResult, DispatchError, Perbill, + traits::{Zero, StaticLookup, Convert}, +}; use frame_support::{ decl_storage, decl_event, ensure, decl_module, decl_error, weights::SimpleDispatchInfo, traits::{ @@ -637,7 +640,7 @@ impl Module { let voters_and_votes = >::enumerate() .map(|(v, i)| (v, i)) .collect::)>>(); - let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote>( + let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote, Perbill>( num_to_elect, 0, candidates, @@ -664,7 +667,7 @@ impl Module { .filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } ) .collect::>(); - let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote>( + let support_map = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>( &new_set, &phragmen_result.assignments, Self::locked_stake_of, diff --git a/frame/elections/src/mock.rs b/frame/elections/src/mock.rs index 1c17703300..37c1dafee6 100644 --- a/frame/elections/src/mock.rs +++ b/frame/elections/src/mock.rs @@ -55,11 +55,11 @@ impl frame_system::Trait for Test { type Version = (); type ModuleToIndex = (); type Doughnut = (); - type DelegatedDispatchVerifier = (); + type DelegatedDispatchVerifier = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/evm/Cargo.toml b/frame/evm/Cargo.toml index b45bf6e31e..66c809fa44 100644 --- a/frame/evm/Cargo.toml +++ b/frame/evm/Cargo.toml @@ -16,9 +16,9 @@ sp-core = { version = "2.0.0", default-features = false, path = "../../primitive sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } -primitive-types = { version = "0.6", default-features = false, features = ["rlp"] } +primitive-types = { version = "0.6.2", default-features = false, features = ["rlp"] } rlp = { version = "0.4", default-features = false } -evm = { version = "0.14", default-features = false } +evm = { version = "0.15", default-features = false } sha3 = { version = "0.8", default-features = false } [features] diff --git a/frame/evm/src/lib.rs b/frame/evm/src/lib.rs index edd3d46a9a..a12d06c69b 100644 --- a/frame/evm/src/lib.rs +++ b/frame/evm/src/lib.rs @@ -25,7 +25,7 @@ pub use crate::backend::{Account, Log, Vicinity, Backend}; use sp_std::{vec::Vec, marker::PhantomData}; use frame_support::{ensure, decl_module, decl_storage, decl_event, decl_error}; -use frame_support::weights::{Weight, WeighData, ClassifyDispatch, DispatchClass, PaysFee}; +use frame_support::weights::{Weight, DispatchClass, FunctionOf}; use frame_support::traits::{Currency, WithdrawReason, ExistenceRequirement}; use frame_system::{self as system, ensure_signed}; use sp_runtime::ModuleId; @@ -34,6 +34,7 @@ use sp_core::{U256, H256, H160, Hasher}; use sp_runtime::{ DispatchResult, traits::{UniqueSaturatedInto, AccountIdConversion, SaturatedConversion}, }; +use sha3::{Digest, Keccak256}; use evm::{ExitReason, ExitSucceed, ExitError}; use evm::executor::StackExecutor; use evm::backend::ApplyBackend; @@ -115,38 +116,6 @@ impl Precompiles for () { } } -struct WeightForCallCreate; - -impl WeighData<(&H160, &Vec, &U256, &u32, &U256)> for WeightForCallCreate { - fn weigh_data( - &self, - (_, _, _, gas_provided, gas_price): (&H160, &Vec, &U256, &u32, &U256) - ) -> Weight { - (*gas_price).saturated_into::().saturating_mul(*gas_provided) - } -} - -impl WeighData<(&Vec, &U256, &u32, &U256)> for WeightForCallCreate { - fn weigh_data( - &self, - (_, _, gas_provided, gas_price): (&Vec, &U256, &u32, &U256) - ) -> Weight { - (*gas_price).saturated_into::().saturating_mul(*gas_provided) - } -} - -impl ClassifyDispatch for WeightForCallCreate { - fn classify_dispatch(&self, _: T) -> DispatchClass { - DispatchClass::Normal - } -} - -impl PaysFee for WeightForCallCreate { - fn pays_fee(&self, _: T) -> bool { - true - } -} - /// EVM module trait pub trait Trait: frame_system::Trait + pallet_timestamp::Trait { /// Calculator for current gas price. @@ -197,6 +166,8 @@ decl_error! { ExitReasonRevert, /// Call returned VM fatal error ExitReasonFatal, + /// Nonce is invalid + InvalidNonce, } } @@ -250,7 +221,7 @@ decl_module! { } /// Issue an EVM call operation. This is similar to a message call transaction in Ethereum. - #[weight = WeightForCallCreate] + #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&H160, &Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] fn call( origin, target: H160, @@ -258,117 +229,99 @@ decl_module! { value: U256, gas_limit: u32, gas_price: U256, + nonce: Option, ) -> DispatchResult { let sender = ensure_signed(origin)?; - ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); let source = T::ConvertAccountId::convert_account_id(&sender); - let vicinity = Vicinity { - gas_price, - origin: source, - }; - - let mut backend = Backend::::new(&vicinity); - let mut executor = StackExecutor::new_with_precompile( - &backend, - gas_limit as usize, - &backend::GASOMETER_CONFIG, - T::Precompiles::execute, - ); - - let total_fee = gas_price.checked_mul(U256::from(gas_limit)) - .ok_or(Error::::FeeOverflow)?; - if Accounts::get(&source).balance < - value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)? - { - Err(Error::::BalanceLow)? - } - executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; - - let reason = executor.transact_call( + Self::execute_evm( source, - target, value, - input, - gas_limit as usize, - ); - - let ret = match reason { - ExitReason::Succeed(_) => Ok(()), - ExitReason::Error(_) => Err(Error::::ExitReasonFailed), - ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), - ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), - }; - let actual_fee = executor.fee(gas_price); - executor.deposit(source, total_fee.saturating_sub(actual_fee)); - - let (values, logs) = executor.deconstruct(); - backend.apply(values, logs, true); - - ret.map_err(Into::into) + gas_limit, + gas_price, + nonce, + |executor| ((), executor.transact_call( + source, + target, + value, + input, + gas_limit as usize, + )), + ).map_err(Into::into) } /// Issue an EVM create operation. This is similar to a contract creation transaction in /// Ethereum. - #[weight = WeightForCallCreate] + #[weight = FunctionOf(|(_, _, gas_limit, gas_price, _): (&Vec, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] fn create( origin, init: Vec, value: U256, gas_limit: u32, gas_price: U256, + nonce: Option, ) -> DispatchResult { let sender = ensure_signed(origin)?; - ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); - let source = T::ConvertAccountId::convert_account_id(&sender); - let vicinity = Vicinity { - gas_price, - origin: source, - }; - - let mut backend = Backend::::new(&vicinity); - let mut executor = StackExecutor::new_with_precompile( - &backend, - gas_limit as usize, - &backend::GASOMETER_CONFIG, - T::Precompiles::execute, - ); - - let total_fee = gas_price.checked_mul(U256::from(gas_limit)) - .ok_or(Error::::FeeOverflow)?; - if Accounts::get(&source).balance < - value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)? - { - Err(Error::::BalanceLow)? - } - executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; - - let create_address = executor.create_address(source, evm::CreateScheme::Dynamic); - let reason = executor.transact_create( + let create_address = Self::execute_evm( source, value, - init, - gas_limit as usize, - ); - - let ret = match reason { - ExitReason::Succeed(_) => { - Module::::deposit_event(Event::Created(create_address)); - Ok(()) + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Legacy { caller: source }, + ), executor.transact_create( + source, + value, + init, + gas_limit as usize, + )) }, - ExitReason::Error(_) => Err(Error::::ExitReasonFailed), - ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), - ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), - }; - let actual_fee = executor.fee(gas_price); - executor.deposit(source, total_fee.saturating_sub(actual_fee)); + )?; - let (values, logs) = executor.deconstruct(); - backend.apply(values, logs, true); + Module::::deposit_event(Event::Created(create_address)); + Ok(()) + } - ret.map_err(Into::into) + /// Issue an EVM create2 operation. + #[weight = FunctionOf(|(_, _, _, gas_limit, gas_price, _): (&Vec, &H256, &U256, &u32, &U256, &Option)| (*gas_price).saturated_into::().saturating_mul(*gas_limit), DispatchClass::Normal, true)] + fn create2( + origin, + init: Vec, + salt: H256, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option, + ) -> DispatchResult { + let sender = ensure_signed(origin)?; + let source = T::ConvertAccountId::convert_account_id(&sender); + + let code_hash = H256::from_slice(Keccak256::digest(&init).as_slice()); + let create_address = Self::execute_evm( + source, + value, + gas_limit, + gas_price, + nonce, + |executor| { + (executor.create_address( + evm::CreateScheme::Create2 { caller: source, code_hash, salt }, + ), executor.transact_create2( + source, + value, + init, + salt, + gas_limit as usize, + )) + }, + )?; + + Module::::deposit_event(Event::Created(create_address)); + Ok(()) } } } @@ -405,4 +358,59 @@ impl Module { AccountCodes::remove(address); AccountStorages::remove_prefix(address); } + + /// Execute an EVM operation. + fn execute_evm( + source: H160, + value: U256, + gas_limit: u32, + gas_price: U256, + nonce: Option, + f: F, + ) -> Result> where + F: FnOnce(&mut StackExecutor>) -> (R, ExitReason), + { + ensure!(gas_price >= T::FeeCalculator::min_gas_price(), Error::::GasPriceTooLow); + + let vicinity = Vicinity { + gas_price, + origin: source, + }; + + let mut backend = Backend::::new(&vicinity); + let mut executor = StackExecutor::new_with_precompile( + &backend, + gas_limit as usize, + &backend::GASOMETER_CONFIG, + T::Precompiles::execute, + ); + + let total_fee = gas_price.checked_mul(U256::from(gas_limit)) + .ok_or(Error::::FeeOverflow)?; + let total_payment = value.checked_add(total_fee).ok_or(Error::::PaymentOverflow)?; + let source_account = Accounts::get(&source); + ensure!(source_account.balance >= total_payment, Error::::BalanceLow); + executor.withdraw(source, total_fee).map_err(|_| Error::::WithdrawFailed)?; + + if let Some(nonce) = nonce { + ensure!(source_account.nonce == nonce, Error::::InvalidNonce); + } + + let (retv, reason) = f(&mut executor); + + let ret = match reason { + ExitReason::Succeed(_) => Ok(retv), + ExitReason::Error(_) => Err(Error::::ExitReasonFailed), + ExitReason::Revert(_) => Err(Error::::ExitReasonRevert), + ExitReason::Fatal(_) => Err(Error::::ExitReasonFatal), + }; + + let actual_fee = executor.fee(gas_price); + executor.deposit(source, total_fee.saturating_sub(actual_fee)); + + let (values, logs) = executor.deconstruct(); + backend.apply(values, logs, true); + + ret + } } diff --git a/frame/example/src/lib.rs b/frame/example/src/lib.rs index a6bd7a7fd8..d82f576e4e 100644 --- a/frame/example/src/lib.rs +++ b/frame/example/src/lib.rs @@ -651,7 +651,8 @@ mod tests { // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ - Perbill, testing::Header, + Perbill, + testing::Header, traits::{BlakeTwo256, OnInitialize, OnFinalize, IdentityLookup}, }; @@ -691,7 +692,7 @@ mod tests { type DelegatedDispatchVerifier = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index ad3d9f8cbf..45bc8c8c99 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -484,7 +484,7 @@ mod tests { type TimestampProvider = TimestampProvider; } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Runtime { diff --git a/frame/executive/tests/doughnut_integration.rs b/frame/executive/tests/doughnut_integration.rs index 3a7b027eb3..c6fcda52bd 100644 --- a/frame/executive/tests/doughnut_integration.rs +++ b/frame/executive/tests/doughnut_integration.rs @@ -126,7 +126,7 @@ impl DoughnutRuntime for Runtime { type TimestampProvider = TimestampProvider; } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const TransferFee: u64 = 0; pub const CreationFee: u64 = 0; } diff --git a/frame/generic-asset/src/lib.rs b/frame/generic-asset/src/lib.rs index 24fad0499c..d2afb1f4ae 100644 --- a/frame/generic-asset/src/lib.rs +++ b/frame/generic-asset/src/lib.rs @@ -157,7 +157,7 @@ use codec::{Decode, Encode, HasCompact, Input, Output, Error as CodecError}; use sp_runtime::{RuntimeDebug, DispatchResult, DispatchError}; use sp_runtime::traits::{ - Bounded, CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Member, One, Saturating, SimpleArithmetic, Zero, + Bounded, CheckedAdd, CheckedSub, MaybeSerializeDeserialize, Member, One, Saturating, AtLeast32Bit, Zero, }; use sp_std::prelude::*; @@ -166,7 +166,7 @@ use frame_support::{ decl_event, decl_module, decl_storage, ensure, decl_error, traits::{ Currency, ExistenceRequirement, Imbalance, LockIdentifier, LockableCurrency, ReservableCurrency, - SignedImbalance, TryDrop, UpdateBalanceOutcome, WithdrawReason, WithdrawReasons, + SignedImbalance, UpdateBalanceOutcome, WithdrawReason, WithdrawReasons, TryDrop, }, additional_traits::{AssetIdAuthority, DummyDispatchVerifier, InherentAssetIdProvider}, Parameter, StorageMap, @@ -182,24 +182,24 @@ pub use self::imbalances::{NegativeImbalance, PositiveImbalance}; pub trait Trait: frame_system::Trait { type Balance: Parameter + Member - + SimpleArithmetic + + AtLeast32Bit + Default + Copy + MaybeSerializeDeserialize + Debug; - type AssetId: Parameter + Member + SimpleArithmetic + Default + Copy; + type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy; type Event: From> + Into<::Event>; } pub trait Subtrait: frame_system::Trait { type Balance: Parameter + Member - + SimpleArithmetic + + AtLeast32Bit + Default + Copy + MaybeSerializeDeserialize + Debug; - type AssetId: Parameter + Member + SimpleArithmetic + Default + Copy; + type AssetId: Parameter + Member + AtLeast32Bit + Default + Copy; } impl Subtrait for T { @@ -1092,10 +1092,10 @@ impl frame_system::Trait for ElevatedTrait { type MaximumBlockLength = T::MaximumBlockLength; type AvailableBlockRatio = T::AvailableBlockRatio; type BlockHashCount = T::BlockHashCount; - type Doughnut = T::Doughnut; - type DelegatedDispatchVerifier = DummyDispatchVerifier; type Version = T::Version; type ModuleToIndex = (); + type Doughnut = T::Doughnut; + type DelegatedDispatchVerifier = DummyDispatchVerifier; } impl Trait for ElevatedTrait { type Balance = T::Balance; diff --git a/frame/grandpa/src/lib.rs b/frame/grandpa/src/lib.rs index 71440f7aac..6164e5ab4b 100644 --- a/frame/grandpa/src/lib.rs +++ b/frame/grandpa/src/lib.rs @@ -34,7 +34,7 @@ use sp_std::prelude::*; use codec::{self as codec, Encode, Decode}; use frame_support::{decl_event, decl_storage, decl_module, decl_error, storage}; use sp_runtime::{ - DispatchResult, generic::{DigestItem, OpaqueDigestItemId}, traits::Zero, Perbill, + DispatchResult, generic::{DigestItem, OpaqueDigestItemId}, traits::Zero, Perbill, PerThing, }; use sp_staking::{ SessionIndex, diff --git a/frame/identity/src/benchmarking.rs b/frame/identity/src/benchmarking.rs new file mode 100644 index 0000000000..13cc2188a1 --- /dev/null +++ b/frame/identity/src/benchmarking.rs @@ -0,0 +1,592 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Identity pallet benchmarking. + +use super::*; + +use frame_system::RawOrigin; +use sp_io::hashing::blake2_256; +use sp_runtime::{BenchmarkResults, BenchmarkParameter, selected_benchmark}; +use sp_runtime::traits::{Bounded, Benchmarking, BenchmarkingSetup, Dispatchable}; + +use crate::Module as Identity; + +// The maximum number of identity registrars we will test. +const MAX_REGISTRARS: u32 = 50; + +// Support Functions +fn account(name: &'static str, index: u32) -> T::AccountId { + let entropy = (name, index).using_encoded(blake2_256); + T::AccountId::decode(&mut &entropy[..]).unwrap_or_default() +} + +// Adds `r` registrars to the Identity Pallet. These registrars will have set fees and fields. +fn add_registrars(r: u32) -> Result<(), &'static str> { + for i in 0..r { + let _ = T::Currency::make_free_balance_be(&account::("registrar", i), BalanceOf::::max_value()); + Identity::::add_registrar(RawOrigin::Root.into(), account::("registrar", i))?; + Identity::::set_fee(RawOrigin::Signed(account::("registrar", i)).into(), i.into(), 10.into())?; + let fields = IdentityFields( + IdentityField::Display | IdentityField::Legal | IdentityField::Web | IdentityField::Riot + | IdentityField::Email | IdentityField::PgpFingerprint | IdentityField::Image | IdentityField::Twitter + ); + Identity::::set_fields(RawOrigin::Signed(account::("registrar", i)).into(), i.into(), fields)?; + } + + assert_eq!(Registrars::::get().len(), r as usize); + Ok(()) +} + +// Adds `s` sub-accounts to the identity of `who`. Each wil have 32 bytes of raw data added to it. +// This additionally returns the vector of sub-accounts to it can be modified if needed. +fn add_sub_accounts(who: T::AccountId, s: u32) -> Result, &'static str> { + let mut subs = Vec::new(); + let who_origin = RawOrigin::Signed(who.clone()); + let data = Data::Raw(vec![0; 32]); + + for i in 0..s { + let sub_account = account::("sub", i); + subs.push((sub_account, data.clone())); + } + Identity::::set_subs(who_origin.into(), subs.clone())?; + + return Ok(subs) +} + +// This creates an `IdentityInfo` object with `num_fields` extra fields. +// All data is pre-populated with some arbitrary bytes. +fn create_identity_info(num_fields: u32) -> IdentityInfo { + let data = Data::Raw(vec![0; 32]); + + let info = IdentityInfo { + additional: vec![(data.clone(), data.clone()); num_fields as usize], + display: data.clone(), + legal: data.clone(), + web: data.clone(), + riot: data.clone(), + email: data.clone(), + pgp_fingerprint: Some([0; 20]), + image: data.clone(), + twitter: data.clone(), + }; + + return info +} + +// Benchmark `add_registrar` extrinsic. +struct AddRegistrar; +impl BenchmarkingSetup, RawOrigin> for AddRegistrar { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Add r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Return the `add_registrar` r + 1 call + Ok((crate::Call::::add_registrar(account::("registrar", r + 1)), RawOrigin::Root)) + } +} + +// Benchmark `set_identity` extrinsic. +struct SetIdentity; +impl BenchmarkingSetup, RawOrigin> for SetIdentity { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()) + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Add r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // The target user + let caller = account::("caller", r); + let caller_lookup: ::Source = T::Lookup::unlookup(caller.clone()); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Add an initial identity + let initial_info = benchmarking::create_identity_info::(1); + Identity::::set_identity(caller_origin.clone(), initial_info)?; + + // User requests judgement from all the registrars, and they approve + for i in 0..r { + Identity::::request_judgement(caller_origin.clone(), i, 10.into())?; + Identity::::provide_judgement( + RawOrigin::Signed(account::("registrar", i)).into(), + i, + caller_lookup.clone(), + Judgement::Reasonable + )?; + } + + // Create identity info with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + // 32 byte data that we reuse below + let info = benchmarking::create_identity_info::(x); + + // Return the `set_identity` call + Ok((crate::Call::::set_identity(info), RawOrigin::Signed(caller))) + } +} + +// Benchmark `set_subs` extrinsic. +struct SetSubs; +impl BenchmarkingSetup, RawOrigin> for SetSubs { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Subs Count + (BenchmarkParameter::S, 1, T::MaxSubAccounts::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Generic data to be used. + let data = Data::Raw(vec![0; 32]); + + // The target user + let caller = account::("caller", 0); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Create their main identity + let info = benchmarking::create_identity_info::(1); + Identity::::set_identity(caller_origin.clone(), info)?; + + // Give them s many sub accounts + let s = components.iter().find(|&c| c.0 == BenchmarkParameter::S).unwrap().1; + let mut subs = add_sub_accounts::(caller.clone(), s)?; + + // Create an s+1 sub account to add + subs.push((account::("sub", s+1), data)); + + // Return the `set_subs` call + Ok((crate::Call::::set_subs(subs), RawOrigin::Signed(caller))) + } +} + +// Benchmark `clear_identity` extrinsic. +struct ClearIdentity; +impl BenchmarkingSetup, RawOrigin> for ClearIdentity { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Subs Count + (BenchmarkParameter::S, 1, T::MaxSubAccounts::get()), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let caller_lookup: ::Source = T::Lookup::unlookup(caller.clone()); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Create their main identity with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + let info = benchmarking::create_identity_info::(x); + Identity::::set_identity(caller_origin.clone(), info)?; + + // Give them s many sub accounts + let s = components.iter().find(|&c| c.0 == BenchmarkParameter::S).unwrap().1; + let _ = benchmarking::add_sub_accounts::(caller.clone(), s)?; + + // User requests judgement from all the registrars, and they approve + for i in 0..r { + Identity::::request_judgement(caller_origin.clone(), i, 10.into())?; + Identity::::provide_judgement( + RawOrigin::Signed(account::("registrar", i)).into(), + i, + caller_lookup.clone(), + Judgement::Reasonable + )?; + } + + // Return the `clear_identity` call + Ok((crate::Call::::clear_identity(), RawOrigin::Signed(caller))) + } +} + +// Benchmark `request_judgement` extrinsic. +struct RequestJudgement; +impl BenchmarkingSetup, RawOrigin> for RequestJudgement { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Create their main identity with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + let info = benchmarking::create_identity_info::(x); + Identity::::set_identity(caller_origin.clone(), info)?; + + // Return the `request_judgement` call + Ok((crate::Call::::request_judgement(r-1, 10.into()), RawOrigin::Signed(caller))) + } +} + +// Benchmark `cancel_request` extrinsic. +struct CancelRequest; +impl BenchmarkingSetup, RawOrigin> for CancelRequest { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Create their main identity with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + let info = benchmarking::create_identity_info::(x); + Identity::::set_identity(caller_origin.clone(), info)?; + + // Request judgement + Identity::::request_judgement(caller_origin.clone(), r-1, 10.into())?; + + Ok((crate::Call::::cancel_request(r-1), RawOrigin::Signed(caller))) + } +} + +// Benchmark `set_fee` extrinsic. +struct SetFee; +impl BenchmarkingSetup, RawOrigin> for SetFee { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Add caller as registrar + Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; + + // Return `set_fee` call + Ok((crate::Call::::set_fee(r, 10.into()), RawOrigin::Signed(caller))) + } +} + +// Benchmark `set_account_id` extrinsic. +struct SetAccountId; +impl BenchmarkingSetup, RawOrigin> for SetAccountId { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Add caller as registrar + Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; + + // Return `set_account_id` call + Ok((crate::Call::::set_account_id(r, account::("new", 0)), RawOrigin::Signed(caller))) + } +} + +// Benchmark `set_fields` extrinsic. +struct SetFields; +impl BenchmarkingSetup, RawOrigin> for SetFields { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Add caller as registrar + Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; + + let fields = IdentityFields( + IdentityField::Display | IdentityField::Legal | IdentityField::Web | IdentityField::Riot + | IdentityField::Email | IdentityField::PgpFingerprint | IdentityField::Image | IdentityField::Twitter + ); + + // Return `set_account_id` call + Ok((crate::Call::::set_fields(r, fields), RawOrigin::Signed(caller))) + } +} + +// Benchmark `provide_judgement` extrinsic. +struct ProvideJudgement; +impl BenchmarkingSetup, RawOrigin> for ProvideJudgement { + + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // Add r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // The user + let user = account::("user", r); + let user_origin: ::Origin = RawOrigin::Signed(user.clone()).into(); + let user_lookup: ::Source = T::Lookup::unlookup(user.clone()); + let _ = T::Currency::make_free_balance_be(&user, BalanceOf::::max_value()); + + // Create their main identity with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + let info = benchmarking::create_identity_info::(x); + Identity::::set_identity(user_origin.clone(), info)?; + + // The caller registrar + let caller = account::("caller", r); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Add caller as registrar + Identity::::add_registrar(RawOrigin::Root.into(), caller.clone())?; + + // User requests judgement from caller registrar + Identity::::request_judgement(user_origin.clone(), r, 10.into())?; + + // Return `provide_judgement` call + Ok((crate::Call::::provide_judgement( + r, + user_lookup.clone(), + Judgement::Reasonable + ), RawOrigin::Signed(caller))) + } +} + +// Benchmark `kill_identity` extrinsic. +struct KillIdentity; +impl BenchmarkingSetup, RawOrigin> for KillIdentity { + + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Registrar Count + (BenchmarkParameter::R, 1, MAX_REGISTRARS), + // Subs Count + (BenchmarkParameter::S, 1, T::MaxSubAccounts::get()), + // Additional Field Count + (BenchmarkParameter::X, 1, T::MaxAdditionalFields::get()), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(crate::Call, RawOrigin), &'static str> + { + // The target user + let caller = account::("caller", 0); + let caller_origin: ::Origin = RawOrigin::Signed(caller.clone()).into(); + let caller_lookup: ::Source = T::Lookup::unlookup(caller.clone()); + let _ = T::Currency::make_free_balance_be(&caller, BalanceOf::::max_value()); + + // Register r registrars + let r = components.iter().find(|&c| c.0 == BenchmarkParameter::R).unwrap().1; + benchmarking::add_registrars::(r)?; + + // Create their main identity with x additional fields + let x = components.iter().find(|&c| c.0 == BenchmarkParameter::X).unwrap().1; + let info = benchmarking::create_identity_info::(x); + Identity::::set_identity(caller_origin.clone(), info)?; + + // Give them s many sub accounts + let s = components.iter().find(|&c| c.0 == BenchmarkParameter::S).unwrap().1; + let _ = benchmarking::add_sub_accounts::(caller.clone(), s)?; + + // User requests judgement from all the registrars, and they approve + for i in 0..r { + Identity::::request_judgement(caller_origin.clone(), i, 10.into())?; + Identity::::provide_judgement( + RawOrigin::Signed(account::("registrar", i)).into(), + i, + caller_lookup.clone(), + Judgement::Reasonable + )?; + } + + // Return the `kill_identity` call + Ok((crate::Call::::kill_identity(caller_lookup), RawOrigin::Root)) + } +} + +// The list of available benchmarks for this pallet. +selected_benchmark!( + AddRegistrar, + SetIdentity, + SetSubs, + ClearIdentity, + RequestJudgement, + CancelRequest, + SetFee, + SetAccountId, + SetFields, + ProvideJudgement, + KillIdentity +); + +impl Benchmarking for Module { + fn run_benchmark(extrinsic: Vec, steps: u32, repeat: u32) -> Result, &'static str> { + // Map the input to the selected benchmark. + let selected_benchmark = match extrinsic.as_slice() { + b"add_registrar" => SelectedBenchmark::AddRegistrar, + b"set_identity" => SelectedBenchmark::SetIdentity, + b"set_subs" => SelectedBenchmark::SetSubs, + b"clear_identity" => SelectedBenchmark::ClearIdentity, + b"request_judgement" => SelectedBenchmark::RequestJudgement, + b"cancel_request" => SelectedBenchmark::CancelRequest, + b"set_fee" => SelectedBenchmark::SetFee, + b"set_account_id" => SelectedBenchmark::SetAccountId, + b"set_fields" => SelectedBenchmark::SetFields, + b"provide_judgement" => SelectedBenchmark::ProvideJudgement, + b"kill_identity" => SelectedBenchmark::KillIdentity, + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + sp_io::benchmarking::commit_db(); + sp_io::benchmarking::wipe_db(); + + // first one is set_identity. + let components = , RawOrigin>>::components(&selected_benchmark); + // results go here + let mut results: Vec = Vec::new(); + // Select the component we will be benchmarking. Each component will be benchmarked. + for (name, low, high) in components.iter() { + // Create up to `STEPS` steps for that component between high and low. + let step_size = ((high - low) / steps).max(1); + let num_of_steps = (high - low) / step_size; + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = low + step_size * s; + + // Select the mid value for all the other components. + let c: Vec<(BenchmarkParameter, u32)> = components.iter() + .map(|(n, l, h)| + (*n, if n == name { component_value } else { (h - l) / 2 + l }) + ).collect(); + + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to benchmark. + let (call, caller) = , RawOrigin>>::instance(&selected_benchmark, &c)?; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + sp_io::benchmarking::commit_db(); + // Run the benchmark. + let start = sp_io::benchmarking::current_time(); + call.dispatch(caller.into())?; + let finish = sp_io::benchmarking::current_time(); + let elapsed = finish - start; + results.push((c.clone(), elapsed)); + // Wipe the DB back to the genesis state. + sp_io::benchmarking::wipe_db(); + } + } + } + return Ok(results); + } +} diff --git a/frame/identity/src/lib.rs b/frame/identity/src/lib.rs index 5d42cce068..d8538fc700 100644 --- a/frame/identity/src/lib.rs +++ b/frame/identity/src/lib.rs @@ -78,6 +78,8 @@ use frame_support::{ }; use frame_system::{self as system, ensure_signed, ensure_root}; +pub mod benchmarking; + type BalanceOf = <::Currency as Currency<::AccountId>>::Balance; type NegativeImbalanceOf = <::Currency as Currency<::AccountId>>::NegativeImbalance; @@ -880,7 +882,7 @@ mod tests { use sp_runtime::traits::BadOrigin; use frame_support::{ assert_ok, assert_noop, impl_outer_origin, parameter_types, weights::Weight, - ord_parameter_types + ord_parameter_types, }; use sp_core::H256; use frame_system::EnsureSignedBy; @@ -922,11 +924,11 @@ mod tests { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); - type DelegatedDispatchVerifier = (); - type Doughnut = (); + type DelegatedDispatchVerifier = (); + type Doughnut = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/im-online/src/lib.rs b/frame/im-online/src/lib.rs index f7067a5843..9a3fc2430c 100644 --- a/frame/im-online/src/lib.rs +++ b/frame/im-online/src/lib.rs @@ -79,7 +79,7 @@ use pallet_session::historical::IdentificationTuple; use sp_runtime::{ offchain::storage::StorageValueRef, RuntimeDebug, - traits::{Convert, Member, Saturating, SimpleArithmetic}, Perbill, + traits::{Convert, Member, Saturating, AtLeast32Bit}, Perbill, PerThing, transaction_validity::{ TransactionValidity, ValidTransaction, InvalidTransaction, TransactionPriority, @@ -102,9 +102,10 @@ pub mod sr25519 { app_crypto!(sr25519, IM_ONLINE); } - /// An i'm online keypair using sr25519 as its crypto. - #[cfg(feature = "std")] - pub type AuthorityPair = app_sr25519::Pair; + sp_application_crypto::with_pair! { + /// An i'm online keypair using sr25519 as its crypto. + pub type AuthorityPair = app_sr25519::Pair; + } /// An i'm online signature using sr25519 as its crypto. pub type AuthoritySignature = app_sr25519::Signature; @@ -119,9 +120,10 @@ pub mod ed25519 { app_crypto!(ed25519, IM_ONLINE); } - /// An i'm online keypair using ed25519 as its crypto. - #[cfg(feature = "std")] - pub type AuthorityPair = app_ed25519::Pair; + sp_application_crypto::with_pair! { + /// An i'm online keypair using ed25519 as its crypto. + pub type AuthorityPair = app_ed25519::Pair; + } /// An i'm online signature using ed25519 as its crypto. pub type AuthoritySignature = app_ed25519::Signature; @@ -151,7 +153,7 @@ struct HeartbeatStatus { pub sent_at: BlockNumber, } -impl HeartbeatStatus { +impl HeartbeatStatus { /// Returns true if heartbeat has been recently sent. /// /// Parameters: diff --git a/frame/indices/src/lib.rs b/frame/indices/src/lib.rs index e6a2a7c144..945095288a 100644 --- a/frame/indices/src/lib.rs +++ b/frame/indices/src/lib.rs @@ -22,7 +22,7 @@ use sp_std::{prelude::*, marker::PhantomData, convert::TryInto}; use codec::{Encode, Codec}; use frame_support::{Parameter, decl_module, decl_event, decl_storage}; -use sp_runtime::traits::{One, SimpleArithmetic, StaticLookup, Member, LookupError}; +use sp_runtime::traits::{One, AtLeast32Bit, StaticLookup, Member, LookupError}; use frame_system::{IsDeadAccount, OnNewAccount}; use self::address::Address as RawAddress; @@ -59,7 +59,7 @@ impl> pub trait Trait: frame_system::Trait { /// Type used for storing an account's index; implies the maximum number of accounts the system /// can hold. - type AccountIndex: Parameter + Member + Codec + Default + SimpleArithmetic + Copy; + type AccountIndex: Parameter + Member + Codec + Default + AtLeast32Bit + Copy; /// Whether an account is dead or not. type IsDeadAccount: IsDeadAccount; diff --git a/frame/nicks/src/lib.rs b/frame/nicks/src/lib.rs index abe97170b0..60e5e3d8b6 100644 --- a/frame/nicks/src/lib.rs +++ b/frame/nicks/src/lib.rs @@ -289,7 +289,7 @@ mod tests { type ModuleToIndex = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/offences/src/mock.rs b/frame/offences/src/mock.rs index 21f508c116..6508aa8d70 100644 --- a/frame/offences/src/mock.rs +++ b/frame/offences/src/mock.rs @@ -89,8 +89,8 @@ impl frame_system::Trait for Runtime { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); - type Doughnut = (); - type DelegatedDispatchVerifier = (); + type Doughnut = (); + type DelegatedDispatchVerifier = (); } impl Trait for Runtime { diff --git a/frame/randomness-collective-flip/src/lib.rs b/frame/randomness-collective-flip/src/lib.rs index fab65626fe..f0d9e12bc0 100644 --- a/frame/randomness-collective-flip/src/lib.rs +++ b/frame/randomness-collective-flip/src/lib.rs @@ -154,7 +154,9 @@ mod tests { use super::*; use sp_core::H256; use sp_runtime::{ - Perbill, traits::{BlakeTwo256, OnInitialize, Header as _, IdentityLookup}, testing::Header, + Perbill, + testing::Header, + traits::{BlakeTwo256, OnInitialize, Header as _, IdentityLookup}, }; use frame_support::{impl_outer_origin, parameter_types, weights::Weight, traits::Randomness}; diff --git a/frame/scored-pool/src/lib.rs b/frame/scored-pool/src/lib.rs index 2ecedd4d3c..d1145e3751 100644 --- a/frame/scored-pool/src/lib.rs +++ b/frame/scored-pool/src/lib.rs @@ -99,7 +99,7 @@ use frame_support::{ }; use frame_system::{self as system, ensure_root, ensure_signed}; use sp_runtime::{ - traits::{EnsureOrigin, SimpleArithmetic, MaybeSerializeDeserialize, Zero, StaticLookup}, + traits::{EnsureOrigin, AtLeast32Bit, MaybeSerializeDeserialize, Zero, StaticLookup}, }; type BalanceOf = <>::Currency as Currency<::AccountId>>::Balance; @@ -121,7 +121,7 @@ pub trait Trait: frame_system::Trait { /// The score attributed to a member or candidate. type Score: - SimpleArithmetic + Clone + Copy + Default + FullCodec + MaybeSerializeDeserialize + Debug; + AtLeast32Bit + Clone + Copy + Default + FullCodec + MaybeSerializeDeserialize + Debug; /// The overarching event type. type Event: From> + Into<::Event>; diff --git a/frame/scored-pool/src/mock.rs b/frame/scored-pool/src/mock.rs index 6260150c73..f002aa5b00 100644 --- a/frame/scored-pool/src/mock.rs +++ b/frame/scored-pool/src/mock.rs @@ -46,7 +46,7 @@ parameter_types! { pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } ord_parameter_types! { @@ -71,8 +71,8 @@ impl frame_system::Trait for Test { type AvailableBlockRatio = AvailableBlockRatio; type Version = (); type ModuleToIndex = (); - type Doughnut = (); - type DelegatedDispatchVerifier = (); + type Doughnut = (); + type DelegatedDispatchVerifier = (); } impl pallet_balances::Trait for Test { diff --git a/frame/session/src/mock.rs b/frame/session/src/mock.rs index eeef6ae4a7..87a4907f49 100644 --- a/frame/session/src/mock.rs +++ b/frame/session/src/mock.rs @@ -21,8 +21,9 @@ use std::cell::RefCell; use frame_support::{impl_outer_origin, parameter_types, weights::Weight}; use sp_core::{crypto::key_types::DUMMY, H256}; use sp_runtime::{ - Perbill, impl_opaque_keys, traits::{BlakeTwo256, IdentityLookup, ConvertInto}, - testing::{Header, UintAuthorityId} + Perbill, impl_opaque_keys, + traits::{BlakeTwo256, IdentityLookup, ConvertInto}, + testing::{Header, UintAuthorityId}, }; use sp_staking::SessionIndex; diff --git a/frame/society/src/lib.rs b/frame/society/src/lib.rs index c1e1803c64..0188ef7cb2 100644 --- a/frame/society/src/lib.rs +++ b/frame/society/src/lib.rs @@ -263,7 +263,7 @@ use frame_support::{decl_error, decl_module, decl_storage, decl_event, ensure, d use frame_support::weights::SimpleDispatchInfo; use frame_support::traits::{ Currency, ReservableCurrency, Randomness, Get, ChangeMembers, - ExistenceRequirement::{KeepAlive, AllowDeath}, + ExistenceRequirement::AllowDeath, }; use frame_system::{self as system, ensure_signed, ensure_root}; @@ -788,7 +788,7 @@ decl_module! { let mut payouts = >::get(&who); if let Some((when, amount)) = payouts.first() { if when <= &>::block_number() { - T::Currency::transfer(&Self::payouts(), &who, *amount, KeepAlive)?; + T::Currency::transfer(&Self::payouts(), &who, *amount, AllowDeath)?; payouts.remove(0); if payouts.is_empty() { >::remove(&who); @@ -1536,18 +1536,24 @@ impl, I: Instance> Module { >::remove_all(); } - // Start a new defender rotation - let phrase = b"society_challenge"; - // we'll need a random seed here. - let seed = T::Randomness::random(phrase); - // seed needs to be guaranteed to be 32 bytes. - let seed = <[u8; 32]>::decode(&mut TrailingZeroInput::new(seed.as_ref())) - .expect("input is padded with zeroes; qed"); - let mut rng = ChaChaRng::from_seed(seed); - let chosen = pick_item(&mut rng, &members).expect("exited if members empty; qed"); - - >::put(&chosen); - Self::deposit_event(RawEvent::Challenged(chosen.clone())); + // Avoid challenging if there's only two members since we never challenge the Head or + // the Founder. + if members.len() > 2 { + // Start a new defender rotation + let phrase = b"society_challenge"; + // we'll need a random seed here. + let seed = T::Randomness::random(phrase); + // seed needs to be guaranteed to be 32 bytes. + let seed = <[u8; 32]>::decode(&mut TrailingZeroInput::new(seed.as_ref())) + .expect("input is padded with zeroes; qed"); + let mut rng = ChaChaRng::from_seed(seed); + let chosen = pick_item(&mut rng, &members[1..members.len() - 1]) + .expect("exited if members empty; qed"); + >::put(&chosen); + Self::deposit_event(RawEvent::Challenged(chosen.clone())); + } else { + >::kill(); + } } } diff --git a/frame/society/src/mock.rs b/frame/society/src/mock.rs index a07f34fe83..31bd645bad 100644 --- a/frame/society/src/mock.rs +++ b/frame/society/src/mock.rs @@ -24,7 +24,9 @@ use sp_core::H256; // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are requried. use sp_runtime::{ - Perbill, traits::{BlakeTwo256, IdentityLookup, OnInitialize, OnFinalize}, testing::Header, + Perbill, + testing::Header, + traits::{BlakeTwo256, IdentityLookup, OnInitialize, OnFinalize}, }; use frame_system::EnsureSignedBy; @@ -51,7 +53,7 @@ parameter_types! { pub const MaximumBlockLength: u32 = 2 * 1024; pub const AvailableBlockRatio: Perbill = Perbill::one(); - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } diff --git a/frame/society/src/tests.rs b/frame/society/src/tests.rs index 3e5afc47f5..47d13d8361 100644 --- a/frame/society/src/tests.rs +++ b/frame/society/src/tests.rs @@ -577,14 +577,14 @@ fn challenges_work() { assert_eq!(Society::defender(), None); // 20 will be challenged during the challenge rotation run_to_block(8); - assert_eq!(Society::defender(), Some(20)); + assert_eq!(Society::defender(), Some(30)); // They can always free vote for themselves - assert_ok!(Society::defender_vote(Origin::signed(20), true)); + assert_ok!(Society::defender_vote(Origin::signed(30), true)); // If no one else votes, nothing happens run_to_block(16); assert_eq!(Society::members(), vec![10, 20, 30, 40]); // New challenge period - assert_eq!(Society::defender(), Some(20)); + assert_eq!(Society::defender(), Some(30)); // Non-member cannot challenge assert_noop!(Society::defender_vote(Origin::signed(1), true), Error::::NotMember); // 3 people say accept, 1 reject @@ -601,7 +601,7 @@ fn challenges_work() { assert_eq!(>::get(30), None); assert_eq!(>::get(40), None); // One more time - assert_eq!(Society::defender(), Some(20)); + assert_eq!(Society::defender(), Some(30)); // 2 people say accept, 2 reject assert_ok!(Society::defender_vote(Origin::signed(10), true)); assert_ok!(Society::defender_vote(Origin::signed(20), true)); @@ -609,10 +609,10 @@ fn challenges_work() { assert_ok!(Society::defender_vote(Origin::signed(40), false)); run_to_block(32); // 20 is suspended - assert_eq!(Society::members(), vec![10, 30, 40]); - assert_eq!(Society::suspended_member(20), true); + assert_eq!(Society::members(), vec![10, 20, 40]); + assert_eq!(Society::suspended_member(30), true); // New defender is chosen - assert_eq!(Society::defender(), Some(40)); + assert_eq!(Society::defender(), Some(20)); // Votes are reset assert_eq!(>::get(10), None); assert_eq!(>::get(20), None); diff --git a/frame/staking/src/inflation.rs b/frame/staking/src/inflation.rs index 9f11fa9845..d5135fcc1f 100644 --- a/frame/staking/src/inflation.rs +++ b/frame/staking/src/inflation.rs @@ -19,7 +19,7 @@ //! The staking rate in NPoS is the total amount of tokens staked by nominators and validators, //! divided by the total token supply. -use sp_runtime::{Perbill, traits::SimpleArithmetic, curve::PiecewiseLinear}; +use sp_runtime::{Perbill, PerThing, traits::AtLeast32Bit, curve::PiecewiseLinear}; /// The total payout to all validators (and their nominators) per era. /// @@ -32,7 +32,7 @@ pub fn compute_total_payout( npos_token_staked: N, total_tokens: N, era_duration: u64 -) -> (N, N) where N: SimpleArithmetic + Clone { +) -> (N, N) where N: AtLeast32Bit + Clone { // Milliseconds per year for the Julian year (365.25 days). const MILLISECONDS_PER_YEAR: u64 = 1000 * 3600 * 24 * 36525 / 100; diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 794ac11ed0..55e52ae038 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -271,12 +271,11 @@ use frame_support::{ }; use pallet_session::historical::SessionManager; use sp_runtime::{ - Perbill, - RuntimeDebug, + Perbill, PerThing, RuntimeDebug, curve::PiecewiseLinear, traits::{ Convert, Zero, One, StaticLookup, CheckedSub, Saturating, Bounded, SaturatedConversion, - SimpleArithmetic, EnsureOrigin, + AtLeast32Bit, EnsureOrigin, } }; use sp_staking::{ @@ -400,7 +399,7 @@ pub struct StakingLedger { impl< AccountId, - Balance: HasCompact + Copy + Saturating + SimpleArithmetic, + Balance: HasCompact + Copy + Saturating + AtLeast32Bit, > StakingLedger { /// Remove entries from `unlocking` that are sufficiently old and reduce the /// total by the sum of their balances. @@ -444,7 +443,7 @@ impl< } impl StakingLedger where - Balance: SimpleArithmetic + Saturating + Copy, + Balance: AtLeast32Bit + Saturating + Copy, { /// Slash the validator for a given amount of balance. This can grow the value /// of the slash in the case that the validator has less than `minimum_balance` @@ -1531,7 +1530,7 @@ impl Module { }); all_nominators.extend(nominator_votes); - let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote>( + let maybe_phragmen_result = sp_phragmen::elect::<_, _, _, T::CurrencyToVote, Perbill>( Self::validator_count() as usize, Self::minimum_validator_count().max(1) as usize, all_validators, @@ -1548,7 +1547,7 @@ impl Module { let to_balance = |e: ExtendedBalance| >>::convert(e); - let supports = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote>( + let supports = sp_phragmen::build_support_map::<_, _, _, T::CurrencyToVote, Perbill>( &elected_stashes, &assignments, Self::slashable_balance_of, diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 75b5a161f7..211bf8f4cd 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -237,7 +237,7 @@ pub struct ExtBuilder { impl Default for ExtBuilder { fn default() -> Self { Self { - existential_deposit: 0, + existential_deposit: 1, validator_pool: false, nominate: true, validator_count: 2, @@ -294,7 +294,7 @@ impl ExtBuilder { pub fn build(self) -> sp_io::TestExternalities { self.set_associated_consts(); let mut storage = frame_system::GenesisConfig::default().build_storage::().unwrap(); - let balance_factor = if self.existential_deposit > 0 { + let balance_factor = if self.existential_deposit > 1 { 256 } else { 1 diff --git a/frame/staking/src/slashing.rs b/frame/staking/src/slashing.rs index df36b1c763..3c8f39501a 100644 --- a/frame/staking/src/slashing.rs +++ b/frame/staking/src/slashing.rs @@ -52,7 +52,7 @@ use super::{ EraIndex, Trait, Module, Store, BalanceOf, Exposure, Perbill, SessionInterface, NegativeImbalanceOf, UnappliedSlash, }; -use sp_runtime::traits::{Zero, Saturating}; +use sp_runtime::{traits::{Zero, Saturating}, PerThing}; use frame_support::{ StorageMap, StorageDoubleMap, traits::{Currency, OnUnbalanced, Imbalance}, diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index f62a58e20b..e59f211b87 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -2066,7 +2066,8 @@ fn slash_in_old_span_does_not_deselect() { ), reporters: vec![], }], - &[Perbill::from_percent(100)], + // NOTE: A 100% slash here would clean up the account, causing de-registration. + &[Perbill::from_percent(95)], 1, ); @@ -2272,7 +2273,7 @@ fn only_slash_for_max_in_era() { #[test] fn garbage_collection_after_slashing() { - ExtBuilder::default().existential_deposit(1).build().execute_with(|| { + ExtBuilder::default().existential_deposit(2).build().execute_with(|| { assert_eq!(Balances::free_balance(11), 256_000); on_offence_now( diff --git a/frame/support/src/additional_traits.rs b/frame/support/src/additional_traits.rs index d747fde976..052a34134b 100644 --- a/frame/support/src/additional_traits.rs +++ b/frame/support/src/additional_traits.rs @@ -8,7 +8,7 @@ use crate::traits::{ use codec::FullCodec; use sp_std::{fmt::Debug, marker::PhantomData, result}; use sp_runtime::traits::{ - PlugDoughnutApi, MaybeSerializeDeserialize, SimpleArithmetic, Zero, + PlugDoughnutApi, MaybeSerializeDeserialize, AtLeast32Bit, Zero, }; /// Perform fee payment for an extrinsic @@ -173,7 +173,7 @@ pub trait MultiCurrencyAccounting { /// The ID type for an account in the system type AccountId: FullCodec + Debug + Default; /// The balance of an account for a particular currency - type Balance: SimpleArithmetic + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default; + type Balance: AtLeast32Bit + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default; /// The ID type of a currency in the system type CurrencyId: FullCodec + Debug + Default; /// A type the is aware of the default network currency ID diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index 922457d398..1c8d049b3d 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -1361,6 +1361,14 @@ macro_rules! decl_module { $call_type::__PhantomItem(_, _) => unreachable!("__PhantomItem should never be used."), } } + + fn get_call_names() -> &'static [&'static str] { + &[ + $( + stringify!($fn_name), + )* + ] + } } // manual implementation of clone/eq/partialeq because using derive erroneously requires @@ -1533,6 +1541,24 @@ macro_rules! impl_outer_dispatch { }, )* } } + + fn get_module_names() -> &'static [&'static str] { + &[$( + stringify!($camelcase), + )*] + } + + fn get_call_names(module: &str) -> &'static [&'static str] { + use $crate::dispatch::{Callable, GetCallName}; + match module { + $( + stringify!($camelcase) => + <<$camelcase as Callable<$runtime>>::Call + as GetCallName>::get_call_names(), + )* + _ => unreachable!(), + } + } } impl $crate::dispatch::Dispatchable for $call_type { type Origin = $origin; @@ -2148,4 +2174,16 @@ mod tests { let expected = CallMetadata { function_name: "aux_3".into(), pallet_name: "Test".into() }; assert_eq!(metadata, expected); } + + #[test] + fn get_call_names() { + let call_names = Call::::get_call_names(); + assert_eq!(["aux_0", "aux_1", "aux_2", "aux_3", "aux_4", "aux_5", "operational"], call_names); + } + + #[test] + fn get_module_names() { + let module_names = OuterCall::get_module_names(); + assert_eq!(["Test"], module_names); + } } diff --git a/frame/support/src/traits.rs b/frame/support/src/traits.rs index ecfababfc0..93c4b55113 100644 --- a/frame/support/src/traits.rs +++ b/frame/support/src/traits.rs @@ -24,7 +24,7 @@ use sp_core::u32_trait::Value as U32; use sp_runtime::{ RuntimeDebug, ConsensusEngineId, DispatchResult, DispatchError, - traits::{MaybeSerializeDeserialize, SimpleArithmetic, Saturating, TrailingZeroInput}, + traits::{MaybeSerializeDeserialize, AtLeast32Bit, Saturating, TrailingZeroInput}, }; use crate::dispatch::Parameter; @@ -267,7 +267,7 @@ pub enum SignedImbalance>{ impl< P: Imbalance, N: Imbalance, - B: SimpleArithmetic + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default, + B: AtLeast32Bit + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default, > SignedImbalance { pub fn zero() -> Self { SignedImbalance::Positive(P::zero()) @@ -330,7 +330,7 @@ impl< /// Abstraction over a fungible assets system. pub trait Currency { /// The balance of an account. - type Balance: SimpleArithmetic + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default; + type Balance: AtLeast32Bit + FullCodec + Copy + MaybeSerializeDeserialize + Debug + Default; /// The opaque token type for an imbalance. This is returned by unbalanced operations /// and must be dealt with. It may be dropped but cannot be cloned. @@ -620,6 +620,10 @@ pub trait VestingSchedule { /// /// If there already exists a vesting schedule for the given account, an `Err` is returned /// and nothing is updated. + /// + /// Is a no-op if the amount to be vested is zero. + /// + /// NOTE: This doesn't alter the free balance of the account. fn add_vesting_schedule( who: &AccountId, locked: >::Balance, @@ -628,6 +632,8 @@ pub trait VestingSchedule { ) -> DispatchResult; /// Remove a vesting schedule for a given account. + /// + /// NOTE: This doesn't alter the free balance of the account. fn remove_vesting_schedule(who: &AccountId); } @@ -653,7 +659,7 @@ bitmask! { } pub trait Time { - type Moment: SimpleArithmetic + Parameter + Default + Copy; + type Moment: AtLeast32Bit + Parameter + Default + Copy + Debug; fn now() -> Self::Moment; } @@ -761,7 +767,10 @@ pub trait Randomness { /// Get a "random" value /// /// Being a deterministic blockchain, real randomness is difficult to come by. This gives you - /// something that approximates it. `subject` is a context identifier and allows you to get a + /// something that approximates it. At best, this will be randomness which was + /// hard to predict a long time ago, but that has become easy to predict recently. + /// + /// `subject` is a context identifier and allows you to get a /// different result to other callers of this function; use it like /// `random(&b"my context"[..])`. fn random(subject: &[u8]) -> Output; @@ -813,12 +822,18 @@ pub struct CallMetadata { /// Gets the function name of the Call. pub trait GetCallName { + /// Return all function names. + fn get_call_names() -> &'static [&'static str]; /// Return the function name of the Call. fn get_call_name(&self) -> &'static str; } /// Gets the metadata for the Call - function name and pallet name. pub trait GetCallMetadata { + /// Return all module names. + fn get_module_names() -> &'static [&'static str]; + /// Return all function names for the given `module`. + fn get_call_names(module: &str) -> &'static [&'static str]; /// Return a [`CallMetadata`], containing function and pallet name of the Call. fn get_call_metadata(&self) -> CallMetadata; } diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index 42a35c49a1..49872ef7db 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -18,7 +18,7 @@ use criterion::{Criterion, criterion_group, criterion_main, black_box}; use frame_system as system; use frame_support::{decl_module, decl_event, impl_outer_origin, impl_outer_event, weights::Weight}; use sp_core::H256; -use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; +use sp_runtime::{Perbill, PerThing, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; mod module { use super::*; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index fe3ee69009..269e989f6b 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -97,16 +97,16 @@ use sp_std::marker::PhantomData; use sp_std::fmt::Debug; use sp_version::RuntimeVersion; use sp_runtime::{ - RuntimeDebug, - generic::{self, Era}, Perbill, DispatchOutcome, DispatchError, + RuntimeDebug, Perbill, DispatchOutcome, DispatchError, + generic::{self, Era}, transaction_validity::{ ValidTransaction, TransactionPriority, TransactionLongevity, TransactionValidityError, InvalidTransaction, TransactionValidity, }, traits::{ - self, CheckEqual, SimpleArithmetic, Zero, SignedExtension, Lookup, LookupError, + self, CheckEqual, AtLeast32Bit, Zero, SignedExtension, Lookup, LookupError, SimpleBitOps, Hash, Member, MaybeDisplay, EnsureOrigin, BadOrigin, SaturatedConversion, - MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, + MaybeSerialize, MaybeSerializeDeserialize, MaybeMallocSizeOf, StaticLookup, One, Bounded, PlugDoughnutApi, }, }; @@ -167,12 +167,12 @@ pub trait Trait: 'static + Eq + Clone { /// Account index (aka nonce) type. This stores the number of previous transactions associated /// with a sender account. type Index: - Parameter + Member + MaybeSerialize + Debug + Default + MaybeDisplay + SimpleArithmetic + Parameter + Member + MaybeSerialize + Debug + Default + MaybeDisplay + AtLeast32Bit + Copy; /// The block number type used by the runtime. type BlockNumber: - Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + SimpleArithmetic + Parameter + Member + MaybeSerializeDeserialize + Debug + MaybeDisplay + AtLeast32Bit + Default + Bounded + Copy + sp_std::hash::Hash + sp_std::str::FromStr + MaybeMallocSizeOf; /// The output of the `Hashing` function. @@ -264,18 +264,20 @@ decl_module! { storage::unhashed::put_raw(well_known_keys::HEAP_PAGES, &pages.encode()); } - /// Set the new runtime code. #[weight = SimpleDispatchInfo::FixedOperational(200_000)] pub fn set_code(origin, code: Vec) { ensure_root(origin)?; + let current_version = T::Version::get(); let new_version = sp_io::misc::runtime_version(&code) .and_then(|v| RuntimeVersion::decode(&mut &v[..]).ok()) .ok_or_else(|| Error::::FailedToExtractRuntimeVersion)?; + if new_version.spec_name != current_version.spec_name { Err(Error::::InvalidSpecName)? } + if new_version.spec_version < current_version.spec_version { Err(Error::::SpecVersionNotAllowedToDecrease)? } else if new_version.spec_version == current_version.spec_version { @@ -285,9 +287,11 @@ decl_module! { Err(Error::::SpecOrImplVersionNeedToIncrease)? } } + storage::unhashed::put_raw(well_known_keys::CODE, &code); Self::deposit_event(Event::CodeUpdated); } + /// Set the new runtime code without doing any checks of the given `code`. #[weight = SimpleDispatchInfo::FixedOperational(200_000)] pub fn set_code_without_checks(origin, code: Vec) { @@ -295,6 +299,7 @@ decl_module! { storage::unhashed::put_raw(well_known_keys::CODE, &code); Self::deposit_event(Event::CodeUpdated); } + /// Set the new changes trie configuration. #[weight = SimpleDispatchInfo::FixedOperational(20_000)] pub fn set_changes_trie_config(origin, changes_trie_config: Option) { @@ -306,6 +311,7 @@ decl_module! { ), None => storage::unhashed::kill(well_known_keys::CHANGES_TRIE_CONFIG), } + let log = generic::DigestItem::ChangesTrieSignal( generic::ChangesTrieSignal::NewConfiguration(changes_trie_config), ); @@ -934,7 +940,7 @@ impl CheckWeight { /// a portion. fn get_dispatch_limit_ratio(class: DispatchClass) -> Perbill { match class { - DispatchClass::Operational => Perbill::one(), + DispatchClass::Operational => ::one(), DispatchClass::Normal => T::AvailableBlockRatio::get(), } } diff --git a/frame/timestamp/Cargo.toml b/frame/timestamp/Cargo.toml index f2d6f40582..e68392f9e9 100644 --- a/frame/timestamp/Cargo.toml +++ b/frame/timestamp/Cargo.toml @@ -9,6 +9,7 @@ license = "GPL-3.0" serde = { version = "1.0.101", optional = true } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0", default-features = false, path = "../../primitives/std" } +sp-io = { version = "2.0.0", default-features = false, path = "../../primitives/io" } sp-runtime = { version = "2.0.0", default-features = false, path = "../../primitives/runtime" } sp-inherents = { version = "2.0.0", default-features = false, path = "../../primitives/inherents" } frame-support = { version = "2.0.0", default-features = false, path = "../support" } diff --git a/frame/timestamp/src/benchmarking.rs b/frame/timestamp/src/benchmarking.rs new file mode 100644 index 0000000000..a391b97cfb --- /dev/null +++ b/frame/timestamp/src/benchmarking.rs @@ -0,0 +1,105 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Timestamp pallet benchmarking. + +use super::*; + +use sp_std::prelude::*; + +use frame_system::RawOrigin; +use sp_runtime::{BenchmarkResults, BenchmarkParameter, selected_benchmark}; +use sp_runtime::traits::{Benchmarking, BenchmarkingSetup, Dispatchable}; + +/// Benchmark `set` extrinsic. +struct Set; +impl BenchmarkingSetup, RawOrigin> for Set { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + vec![ + // Current time ("Now") + (BenchmarkParameter::N, 1, 100), + ] + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(Call, RawOrigin), &'static str> + { + let user_origin = RawOrigin::None; + let now = components.iter().find(|&c| c.0 == BenchmarkParameter::N).unwrap().1; + + // Return the `set` call + Ok((Call::::set(now.into()), user_origin)) + } +} + +selected_benchmark!(Set); + +impl Benchmarking for Module { + fn run_benchmark(extrinsic: Vec, steps: u32, repeat: u32) -> Result, &'static str> { + // Map the input to the selected benchmark. + let selected_benchmark = match extrinsic.as_slice() { + b"set" => SelectedBenchmark::Set, + _ => return Err("Could not find extrinsic."), + }; + + // Warm up the DB + sp_io::benchmarking::commit_db(); + sp_io::benchmarking::wipe_db(); + + let components = , RawOrigin>>::components(&selected_benchmark); + let mut results: Vec = Vec::new(); + + // Select the component we will be benchmarking. Each component will be benchmarked. + for (name, low, high) in components.iter() { + // Create up to `STEPS` steps for that component between high and low. + let step_size = ((high - low) / steps).max(1); + let num_of_steps = (high - low) / step_size; + for s in 0..num_of_steps { + // This is the value we will be testing for component `name` + let component_value = low + step_size * s; + + // Select the mid value for all the other components. + let c: Vec<(BenchmarkParameter, u32)> = components.iter() + .map(|(n, l, h)| + (*n, if n == name { component_value } else { (h - l) / 2 + l }) + ).collect(); + + // Run the benchmark `repeat` times. + for _ in 0..repeat { + // Set up the externalities environment for the setup we want to benchmark. + let (call, caller) = , + RawOrigin, + >>::instance(&selected_benchmark, &c)?; + // Commit the externalities to the database, flushing the DB cache. + // This will enable worst case scenario for reading from the database. + sp_io::benchmarking::commit_db(); + // Run the benchmark. + let start = sp_io::benchmarking::current_time(); + call.dispatch(caller.into())?; + let finish = sp_io::benchmarking::current_time(); + let elapsed = finish - start; + results.push((c.clone(), elapsed)); + // Wipe the DB back to the genesis state. + sp_io::benchmarking::wipe_db(); + } + } + } + + return Ok(results); + } +} diff --git a/frame/timestamp/src/lib.rs b/frame/timestamp/src/lib.rs index c84c485cff..00a50e213f 100644 --- a/frame/timestamp/src/lib.rs +++ b/frame/timestamp/src/lib.rs @@ -90,6 +90,8 @@ #![cfg_attr(not(feature = "std"), no_std)] +mod benchmarking; + use sp_std::{result, cmp}; use sp_inherents::{ProvideInherent, InherentData, InherentIdentifier}; use frame_support::{Parameter, decl_storage, decl_module}; @@ -97,7 +99,7 @@ use frame_support::traits::{Time, Get}; use sp_runtime::{ RuntimeString, traits::{ - SimpleArithmetic, Zero, SaturatedConversion, Scale + AtLeast32Bit, Zero, SaturatedConversion, Scale } }; use frame_support::weights::SimpleDispatchInfo; @@ -110,7 +112,7 @@ use sp_timestamp::{ /// The module configuration trait pub trait Trait: frame_system::Trait { /// Type used for expressing timestamp. - type Moment: Parameter + Default + SimpleArithmetic + type Moment: Parameter + Default + AtLeast32Bit + Scale + Copy; /// Something which can be notified when the timestamp is set. Set this to `()` if not needed. diff --git a/frame/transaction-payment/rpc/runtime-api/src/lib.rs b/frame/transaction-payment/rpc/runtime-api/src/lib.rs index 6e7425f8b8..77b6e3b454 100644 --- a/frame/transaction-payment/rpc/runtime-api/src/lib.rs +++ b/frame/transaction-payment/rpc/runtime-api/src/lib.rs @@ -22,12 +22,13 @@ use sp_std::prelude::*; use frame_support::weights::{Weight, DispatchClass}; use codec::{Encode, Codec, Decode}; #[cfg(feature = "std")] -use serde::{Serialize, Deserialize}; -use sp_runtime::traits::{UniqueSaturatedInto, SaturatedConversion}; +use serde::{Serialize, Deserialize, Serializer, Deserializer}; +use sp_runtime::traits::{MaybeDisplay, MaybeFromStr}; /// Some information related to a dispatchable that can be queried from the runtime. #[derive(Eq, PartialEq, Encode, Decode, Default)] -#[cfg_attr(feature = "std", derive(Debug))] +#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] +#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] pub struct RuntimeDispatchInfo { /// Weight of this dispatch. pub weight: Weight, @@ -35,47 +36,27 @@ pub struct RuntimeDispatchInfo { pub class: DispatchClass, /// The partial inclusion fee of this dispatch. This does not include tip or anything else which /// is dependent on the signature (aka. depends on a `SignedExtension`). + #[cfg_attr(feature = "std", serde(bound(serialize = "Balance: std::fmt::Display")))] + #[cfg_attr(feature = "std", serde(serialize_with = "serialize_as_string"))] + #[cfg_attr(feature = "std", serde(bound(deserialize = "Balance: std::str::FromStr")))] + #[cfg_attr(feature = "std", serde(deserialize_with = "deserialize_from_string"))] pub partial_fee: Balance, } -/// A capped version of `RuntimeDispatchInfo`. -/// -/// The `Balance` is capped (or expanded) to `u64` to avoid serde issues with `u128`. -#[derive(Eq, PartialEq, Encode, Decode, Default)] -#[cfg_attr(feature = "std", derive(Debug, Serialize, Deserialize))] -#[cfg_attr(feature = "std", serde(rename_all = "camelCase"))] -pub struct CappedDispatchInfo { - /// Weight of this dispatch. - pub weight: Weight, - /// Class of this dispatch. - pub class: DispatchClass, - /// The partial inclusion fee of this dispatch. This does not include tip or anything else which - /// is dependent on the signature (aka. depends on a `SignedExtension`). - pub partial_fee: u64, +#[cfg(feature = "std")] +fn serialize_as_string(t: &T, serializer: S) -> Result { + serializer.serialize_str(&t.to_string()) } -impl CappedDispatchInfo { - /// Create a new `CappedDispatchInfo` from `RuntimeDispatchInfo`. - pub fn new>( - dispatch: RuntimeDispatchInfo, - ) -> Self { - let RuntimeDispatchInfo { - weight, - class, - partial_fee, - } = dispatch; - - Self { - weight, - class, - partial_fee: partial_fee.saturated_into(), - } - } +#[cfg(feature = "std")] +fn deserialize_from_string<'de, D: Deserializer<'de>, T: std::str::FromStr>(deserializer: D) -> Result { + let s = String::deserialize(deserializer)?; + s.parse::().map_err(|_| serde::de::Error::custom("Parse from string failed")) } sp_api::decl_runtime_apis! { pub trait TransactionPaymentApi where - Balance: Codec, + Balance: Codec + MaybeDisplay + MaybeFromStr, Extrinsic: Codec, { fn query_info(uxt: Extrinsic, len: u32) -> RuntimeDispatchInfo; @@ -87,18 +68,34 @@ mod tests { use super::*; #[test] - fn should_serialize_properly_with_u64() { + fn should_serialize_and_deserialize_properly_with_string() { let info = RuntimeDispatchInfo { weight: 5, class: DispatchClass::Normal, partial_fee: 1_000_000_u64, }; - let info = CappedDispatchInfo::new(info); - assert_eq!( - serde_json::to_string(&info).unwrap(), - r#"{"weight":5,"class":"normal","partialFee":1000000}"#, - ); + let json_str = r#"{"weight":5,"class":"normal","partialFee":"1000000"}"#; + + assert_eq!(serde_json::to_string(&info).unwrap(), json_str); + assert_eq!(serde_json::from_str::>(json_str).unwrap(), info); + + // should not panic + serde_json::to_value(&info).unwrap(); + } + + #[test] + fn should_serialize_and_deserialize_properly_large_value() { + let info = RuntimeDispatchInfo { + weight: 5, + class: DispatchClass::Normal, + partial_fee: u128::max_value(), + }; + + let json_str = r#"{"weight":5,"class":"normal","partialFee":"340282366920938463463374607431768211455"}"#; + + assert_eq!(serde_json::to_string(&info).unwrap(), json_str); + assert_eq!(serde_json::from_str::>(json_str).unwrap(), info); // should not panic serde_json::to_value(&info).unwrap(); diff --git a/frame/transaction-payment/rpc/src/lib.rs b/frame/transaction-payment/rpc/src/lib.rs index aadc33759e..945b0119d7 100644 --- a/frame/transaction-payment/rpc/src/lib.rs +++ b/frame/transaction-payment/rpc/src/lib.rs @@ -21,21 +21,21 @@ use codec::{Codec, Decode}; use sp_blockchain::HeaderBackend; use jsonrpc_core::{Error as RpcError, ErrorCode, Result}; use jsonrpc_derive::rpc; -use sp_runtime::{generic::BlockId, traits::{Block as BlockT, UniqueSaturatedInto}}; +use sp_runtime::{generic::BlockId, traits::{Block as BlockT, MaybeDisplay, MaybeFromStr}}; use sp_api::ProvideRuntimeApi; use sp_core::Bytes; -use pallet_transaction_payment_rpc_runtime_api::CappedDispatchInfo; +use pallet_transaction_payment_rpc_runtime_api::RuntimeDispatchInfo; pub use pallet_transaction_payment_rpc_runtime_api::TransactionPaymentApi as TransactionPaymentRuntimeApi; pub use self::gen_client::Client as TransactionPaymentClient; #[rpc] -pub trait TransactionPaymentApi { +pub trait TransactionPaymentApi { #[rpc(name = "payment_queryInfo")] fn query_info( &self, encoded_xt: Bytes, at: Option - ) -> Result; + ) -> Result; } /// A struct that implements the [`TransactionPaymentApi`]. @@ -68,20 +68,20 @@ impl From for i64 { } } -impl TransactionPaymentApi<::Hash, Balance> +impl TransactionPaymentApi<::Hash, RuntimeDispatchInfo> for TransactionPayment where Block: BlockT, C: Send + Sync + 'static + ProvideRuntimeApi + HeaderBackend, C::Api: TransactionPaymentRuntimeApi, - Balance: Codec + UniqueSaturatedInto, + Balance: Codec + MaybeDisplay + MaybeFromStr, Extrinsic: Codec + Send + Sync + 'static, { fn query_info( &self, encoded_xt: Bytes, at: Option<::Hash> - ) -> Result { + ) -> Result> { let api = self.client.runtime_api(); let at = BlockId::hash(at.unwrap_or_else(|| // If the block hash is not supplied assume the best block. @@ -99,6 +99,6 @@ where code: ErrorCode::ServerError(Error::RuntimeError.into()), message: "Unable to query dispatch info.".into(), data: Some(format!("{:?}", e).into()), - }).map(CappedDispatchInfo::new) + }) } } diff --git a/frame/transaction-payment/src/lib.rs b/frame/transaction-payment/src/lib.rs index e5353e8540..28870bfc0a 100644 --- a/frame/transaction-payment/src/lib.rs +++ b/frame/transaction-payment/src/lib.rs @@ -300,15 +300,15 @@ mod tests { type MaximumBlockWeight = MaximumBlockWeight; type MaximumBlockLength = MaximumBlockLength; type AvailableBlockRatio = AvailableBlockRatio; - type Doughnut = (); - type DelegatedDispatchVerifier = (); type Version = (); type ModuleToIndex = (); + type Doughnut = (); + type DelegatedDispatchVerifier = (); } parameter_types! { pub const CreationFee: u64 = 0; - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; } impl pallet_balances::Trait for Runtime { @@ -396,14 +396,18 @@ mod tests { self.set_constants(); let mut t = frame_system::GenesisConfig::default().build_storage::().unwrap(); pallet_balances::GenesisConfig:: { - balances: vec![ - (1, 10 * self.balance_factor), - (2, 20 * self.balance_factor), - (3, 30 * self.balance_factor), - (4, 40 * self.balance_factor), - (5, 50 * self.balance_factor), - (6, 60 * self.balance_factor) - ], + balances: if self.balance_factor > 0 { + vec![ + (1, 10 * self.balance_factor), + (2, 20 * self.balance_factor), + (3, 30 * self.balance_factor), + (4, 40 * self.balance_factor), + (5, 50 * self.balance_factor), + (6, 60 * self.balance_factor) + ] + } else { + vec![] + }, }.assimilate_storage(&mut t).unwrap(); t.into() } diff --git a/frame/treasury/src/lib.rs b/frame/treasury/src/lib.rs index 31f93b708c..782a6365cc 100644 --- a/frame/treasury/src/lib.rs +++ b/frame/treasury/src/lib.rs @@ -590,7 +590,7 @@ impl Module { } } - /// Remove any non-members of `Tippers` from a `tips` vectr. `O(T)`. + /// Remove any non-members of `Tippers` from a `tips` vector. `O(T)`. fn retain_active_tips(tips: &mut Vec<(T::AccountId, BalanceOf)>) { let members = T::Tippers::sorted_members(); let mut members_iter = members.iter(); @@ -724,7 +724,9 @@ mod tests { use frame_support::traits::Contains; use sp_core::H256; use sp_runtime::{ - traits::{BlakeTwo256, OnFinalize, IdentityLookup, BadOrigin}, testing::Header, Perbill + Perbill, + testing::Header, + traits::{BlakeTwo256, OnFinalize, IdentityLookup, BadOrigin}, }; impl_outer_origin! { diff --git a/frame/utility/src/lib.rs b/frame/utility/src/lib.rs index a53fce989f..1a8c9a931e 100644 --- a/frame/utility/src/lib.rs +++ b/frame/utility/src/lib.rs @@ -704,7 +704,7 @@ mod tests { type DelegatedDispatchVerifier = (); } parameter_types! { - pub const ExistentialDeposit: u64 = 0; + pub const ExistentialDeposit: u64 = 1; pub const CreationFee: u64 = 0; } impl pallet_balances::Trait for Test { diff --git a/frame/vesting/src/lib.rs b/frame/vesting/src/lib.rs index 967dee61e6..d7e31b66f6 100644 --- a/frame/vesting/src/lib.rs +++ b/frame/vesting/src/lib.rs @@ -50,9 +50,9 @@ use sp_std::prelude::*; use sp_std::fmt::Debug; use codec::{Encode, Decode}; use sp_runtime::{DispatchResult, RuntimeDebug, traits::{ - StaticLookup, Zero, SimpleArithmetic, MaybeSerializeDeserialize, Saturating, Convert + StaticLookup, Zero, AtLeast32Bit, MaybeSerializeDeserialize, Saturating, Convert }}; -use frame_support::{decl_module, decl_event, decl_storage, ensure, decl_error}; +use frame_support::{decl_module, decl_event, decl_storage, decl_error}; use frame_support::traits::{ Currency, LockableCurrency, VestingSchedule, WithdrawReason, LockIdentifier }; @@ -85,8 +85,8 @@ pub struct VestingInfo { } impl< - Balance: SimpleArithmetic + Copy, - BlockNumber: SimpleArithmetic + Copy, + Balance: AtLeast32Bit + Copy, + BlockNumber: AtLeast32Bit + Copy, > VestingInfo { /// Amount locked at block `n`. pub fn locked_at< @@ -212,16 +212,18 @@ impl Module { /// (Re)set or remove the module's currency lock on `who`'s account in accordance with their /// current unvested amount. fn update_lock(who: T::AccountId) -> DispatchResult { - ensure!(Vesting::::contains_key(&who), Error::::NotVesting); - let unvested = Self::vesting_balance(&who); - if unvested.is_zero() { + let vesting = Self::vesting(&who).ok_or(Error::::NotVesting)?; + let now = >::block_number(); + let locked_now = vesting.locked_at::(now); + + if locked_now.is_zero() { T::Currency::remove_lock(VESTING_ID, &who); Vesting::::remove(&who); Self::deposit_event(RawEvent::VestingCompleted(who)); } else { let reasons = WithdrawReason::Transfer | WithdrawReason::Reserve; - T::Currency::set_lock(VESTING_ID, &who, unvested, reasons); - Self::deposit_event(RawEvent::VestingUpdated(who, unvested)); + T::Currency::set_lock(VESTING_ID, &who, locked_now, reasons); + Self::deposit_event(RawEvent::VestingUpdated(who, locked_now)); } Ok(()) } @@ -249,6 +251,10 @@ impl VestingSchedule for Module where /// If there already exists a vesting schedule for the given account, an `Err` is returned /// and nothing is updated. /// + /// On success, a linearly reducing amount of funds will be locked. In order to realise any + /// reduction of the lock over time as it diminishes, the account owner must use `vest` or + /// `vest_other`. + /// /// Is a no-op if the amount to be vested is zero. fn add_vesting_schedule( who: &T::AccountId, @@ -292,7 +298,9 @@ mod tests { // The testing primitives are very useful for avoiding having to work with signatures // or public keys. `u64` is used as the `AccountId` and no `Signature`s are required. use sp_runtime::{ - Perbill, testing::Header, traits::{BlakeTwo256, IdentityLookup, Identity, OnInitialize}, + Perbill, + testing::Header, + traits::{BlakeTwo256, IdentityLookup, Identity, OnInitialize}, }; use sp_storage::Storage; diff --git a/primitives/allocator/src/freeing_bump.rs b/primitives/allocator/src/freeing_bump.rs index f51dc222a2..caac9dd6c4 100644 --- a/primitives/allocator/src/freeing_bump.rs +++ b/primitives/allocator/src/freeing_bump.rs @@ -16,42 +16,40 @@ //! This module implements a freeing-bump allocator. //! -//! The algorithm is as follows: -//! We store `N` linked list heads, where `N` is the total number of sizes -//! of allocations to support. A simple set is powers of two from 8 bytes -//! to 16,777,216 bytes (2^3 - 2^24 inclusive), resulting in `N = 22`: +//! The heap is a continuous linear memory and chunks are allocated using a bump allocator. //! //! ```ignore -//! let mut heads [u64; N] = [0; N]; -//! fn size(n: u64) -> u64 { 8 << n } -//! let mut bumper = 0; -//! fn bump(n: u64) -> u64 { let res = bumper; bumper += n; res } +//! +-------------+-------------------------------------------------+ +//! | | | +//! +-------------+-------------------------------------------------+ +//! ^ +//! |_ bumper //! ``` //! -//! We assume there is a slab of heap to be allocated: +//! Only allocations with sizes of power of two can be allocated. If the incoming request has a non +//! power of two size it is increased to the nearest power of two. The power of two of size is +//! referred as **an order**. //! -//! ```ignore -//! let mut heap = [0u8; HEAP_SIZE]; -//! ``` +//! Each allocation has a header immediately preceding to it. The header is always 8 bytes and can +//! be of two types: free and occupied. //! -//! Whenever we allocate, we select the lowest linked list item size that -//! will fit the allocation (i.e. the next highest power of two). -//! We then check to see if the linked list is empty. If empty, we use -//! the bump allocator to get the allocation with an extra 8 bytes -//! preceding it. We initialise those preceding 8 bytes to identify the -//! list to which it belongs. If it is not empty, we unlink the first item from -//! the linked list and then reset the 8 preceding bytes so they now record -//! the identity of the linked list. +//! For implementing freeing we maintain a linked lists for each order. The maximum supported +//! allocation size is capped, therefore the number of orders and thus the linked lists is as well +//! limited. //! -//! To deallocate we use the preceding 8 bytes of the allocation to knit -//! back the allocation into the linked list from the head. +//! When the allocater serves an allocation request it first checks the linked list for the respective +//! order. If it doesn't have any free chunks, the allocator requests memory from the bump allocator. +//! In any case the order is stored in the header of the allocation. +//! +//! Upon deallocation we get the order of the allocation from its header and then add that +//! allocation to the linked list for the respective order. use crate::Error; -use sp_std::{convert::{TryFrom, TryInto}, ops::Range}; +use sp_std::{convert::{TryFrom, TryInto}, ops::{Range, Index, IndexMut}}; use sp_wasm_interface::{Pointer, WordSize}; -// The pointers need to be aligned to 8 bytes. This is because the -// maximum value type handled by wasm32 is u64. +/// The minimal alignment guaranteed by this allocator. The alignment of 8 is choosen because it is +/// the alignment guaranteed by wasm32. const ALIGNMENT: u32 = 8; // The pointer returned by `allocate()` needs to fulfill the alignment @@ -65,17 +63,7 @@ const MIN_POSSIBLE_ALLOCATION: u32 = 8; // Each pointer is prefixed with 8 bytes, which identify the list index // to which it belongs. -const PREFIX_SIZE: u32 = 8; - -/// An implementation of freeing bump allocator. -/// -/// Refer to the module-level documentation for further details. -pub struct FreeingBumpHeapAllocator { - bumper: u32, - heads: [u32; N], - ptr_offset: u32, - total_size: u32, -} +const HEADER_SIZE: u32 = 8; /// Create an allocator error. fn error(msg: &'static str) -> Error { @@ -93,6 +81,219 @@ macro_rules! trace { } } +/// The exponent for the power of two sized block adjusted to the minimum size. +/// +/// This way, if `MIN_POSSIBLE_ALLOCATION == 8`, we would get: +/// +/// power_of_two_size | order +/// 8 | 0 +/// 16 | 1 +/// 32 | 2 +/// 64 | 3 +/// +/// and so on. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +struct Order(u32); + +impl Order { + /// Create `Order` object from a raw order. + /// + /// Returns `Err` if it is greater than the maximum supported order. + fn from_raw(order: u32) -> Result { + if order < N as u32 { + Ok(Self(order)) + } else { + Err(error("invalid order")) + } + } + + /// Compute the order by the given size + /// + /// The size is clamped, so that the following holds: + /// + /// `MIN_POSSIBLE_ALLOCATION <= size <= MAX_POSSIBLE_ALLOCATION` + fn from_size(size: u32) -> Result { + let clamped_size = if size > MAX_POSSIBLE_ALLOCATION { + return Err(Error::RequestedAllocationTooLarge); + } else if size < MIN_POSSIBLE_ALLOCATION { + MIN_POSSIBLE_ALLOCATION + } else { + size + }; + + // Round the clamped size to the next power of two. + // + // It returns the unchanged value if the value is already a power of two. + let power_of_two_size = clamped_size.next_power_of_two(); + + // Compute the number of trailing zeroes to get the order. We adjust it by the number of + // trailing zeroes in the minimum possible allocation. + let order = power_of_two_size.trailing_zeros() - MIN_POSSIBLE_ALLOCATION.trailing_zeros(); + + Ok(Self(order)) + } + + /// Returns the corresponding size for this order. + /// + /// Note that it is always a power of two. + fn size(&self) -> u32 { + MIN_POSSIBLE_ALLOCATION << self.0 + } + + /// Extract the order as `u32`. + fn into_raw(self) -> u32 { + self.0 + } +} + +/// A marker for denoting the end of the linked list. +const EMPTY_MARKER: u32 = u32::max_value(); + +/// A link between headers in the free list. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Link { + /// Null, denotes that there is no next element. + Null, + /// Link to the next element represented as a pointer to the a header. + Ptr(u32), +} + +impl Link { + /// Creates a link from raw value. + fn from_raw(raw: u32) -> Self { + if raw != EMPTY_MARKER { + Self::Ptr(raw) + } else { + Self::Null + } + } + + /// Converts this link into a raw u32. + fn into_raw(self) -> u32 { + match self { + Self::Null => EMPTY_MARKER, + Self::Ptr(ptr) => ptr, + } + } +} + +/// A header of an allocation. +/// +/// The header is encoded in memory as follows. +/// +/// ## Free header +/// +/// ```ignore +/// 64 32 0 +// +--------------+-------------------+ +/// | 0 | next element link | +/// +--------------+-------------------+ +/// ``` +/// +/// ## Occupied header +/// +/// ```ignore +/// 64 32 0 +// +--------------+-------------------+ +/// | 1 | order | +/// +--------------+-------------------+ +/// ``` +#[derive(Clone, Debug, PartialEq, Eq)] +enum Header { + /// A free header contains a link to the next element to form a free linked list. + Free(Link), + /// An occupied header has attached order to know in which free list we should put the + /// allocation upon deallocation. + Occupied(Order), +} + +impl Header { + fn read_from(memory: &M, header_ptr: u32) -> Result { + let raw_header = memory.read_le_u64(header_ptr)?; + + // Check if the header represents an occupied or free allocation and extract the header data + // by trimming (and discarding) the high bits. + let occupied = raw_header & 0x00000001_00000000 != 0; + let header_data = raw_header as u32; + + Ok(if occupied { + Self::Occupied(Order::from_raw(header_data)?) + } else { + Self::Free(Link::from_raw(header_data)) + }) + } + + /// Write out this header to memory. + fn write_into(&self, memory: &mut M, header_ptr: u32) -> Result<(), Error> { + let (header_data, occupied_mask) = match *self { + Self::Occupied(order) => (order.into_raw(), 0x00000001_00000000), + Self::Free(link) => (link.into_raw(), 0x00000000_00000000), + }; + let raw_header = header_data as u64 | occupied_mask; + memory.write_le_u64(header_ptr, raw_header)?; + Ok(()) + } + + /// Returns the order of the allocation if this is an occupied header. + fn into_occupied(self) -> Option { + match self { + Self::Occupied(order) => Some(order), + _ => None, + } + } + + /// Returns the link to the next element in the free list if this is a free header. + fn into_free(self) -> Option { + match self { + Self::Free(link) => Some(link), + _ => None, + } + } +} + +/// This struct represents a collection of intrusive linked lists for each order. +struct FreeLists { + heads: [Link; N], +} + +impl FreeLists { + /// Creates the free empty lists. + fn new() -> Self { + Self { + heads: [Link::Null; N] + } + } + + /// Replaces a given link for the specified order and returns the old one. + fn replace(&mut self, order: Order, new: Link) -> Link { + let prev = self[order]; + self[order] = new; + prev + } +} + +impl Index for FreeLists { + type Output = Link; + fn index(&self, index: Order) -> &Link { + &self.heads[index.0 as usize] + } +} + +impl IndexMut for FreeLists { + fn index_mut(&mut self, index: Order) -> &mut Link { + &mut self.heads[index.0 as usize] + } +} + +/// An implementation of freeing bump allocator. +/// +/// Refer to the module-level documentation for further details. +pub struct FreeingBumpHeapAllocator { + bumper: u32, + free_lists: FreeLists, + total_size: u32, +} + impl FreeingBumpHeapAllocator { /// Creates a new allocation heap which follows a freeing-bump strategy. /// The maximum size which can be allocated at once is 16 MiB. @@ -101,13 +302,11 @@ impl FreeingBumpHeapAllocator { /// /// - `heap_base` - the offset from the beginning of the linear memory where the heap starts. pub fn new(heap_base: u32) -> Self { - // ptr_offset is the next alignment boundary on or after heap_base. - let ptr_offset = (heap_base + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT; + let aligned_heap_base = (heap_base + ALIGNMENT - 1) / ALIGNMENT * ALIGNMENT; FreeingBumpHeapAllocator { - bumper: 0, - heads: [u32::max_value(); N], - ptr_offset, + bumper: aligned_heap_base, + free_lists: FreeLists::new(), total_size: 0, } } @@ -122,46 +321,42 @@ impl FreeingBumpHeapAllocator { /// /// - `mem` - a slice representing the linear memory on which this allocator operates. /// - `size` - size in bytes of the allocation request - pub fn allocate(&mut self, mem: &mut [u8], size: WordSize) -> Result, Error> { - let mem_size = u32::try_from(mem.len()) - .expect("size of Wasm linear memory is <2^32; qed"); - let max_heap_size = mem_size - self.ptr_offset; - - if size > MAX_POSSIBLE_ALLOCATION { - return Err(Error::RequestedAllocationTooLarge); - } - - let size = size.max(MIN_POSSIBLE_ALLOCATION); - let item_size = size.next_power_of_two(); - if item_size + PREFIX_SIZE + self.total_size > max_heap_size { - return Err(Error::AllocatorOutOfSpace); - } - - let list_index = (item_size.trailing_zeros() - 3) as usize; - let ptr: u32 = if self.heads[list_index] != u32::max_value() { - // Something from the free list - let ptr = self.heads[list_index]; - assert!( - ptr + item_size + PREFIX_SIZE <= max_heap_size, - "Pointer is looked up in list of free entries, into which - only valid values are inserted; qed" - ); - - self.heads[list_index] = self.get_heap_u64(mem, ptr)? - .try_into() - .map_err(|_| error("read invalid free list pointer"))?; - ptr - } else { - // Nothing to be freed. Bump. - self.bump(item_size, max_heap_size)? + pub fn allocate( + &mut self, + mem: &mut M, + size: WordSize, + ) -> Result, Error> { + let order = Order::from_size(size)?; + + let header_ptr: u32 = match self.free_lists[order] { + Link::Ptr(header_ptr) => { + assert!( + header_ptr + order.size() + HEADER_SIZE <= mem.size(), + "Pointer is looked up in list of free entries, into which + only valid values are inserted; qed" + ); + + // Remove this header from the free list. + let next_free = Header::read_from(mem, header_ptr)? + .into_free() + .ok_or_else(|| error("free list points to a occupied header"))?; + self.free_lists[order] = next_free; + + header_ptr + } + Link::Null => { + // Corresponding free list is empty. Allocate a new item. + self.bump(order.size() + HEADER_SIZE, mem.size())? + } }; - self.set_heap_u64(mem, ptr, list_index as u64)?; + // Write the order in the occupied header. + Header::Occupied(order).write_into(mem, header_ptr)?; - self.total_size = self.total_size + item_size + PREFIX_SIZE; + self.total_size += order.size() + HEADER_SIZE; trace!("Heap size is {} bytes after allocation", self.total_size); - Ok(Pointer::new(self.ptr_offset + ptr + PREFIX_SIZE)) + Ok(Pointer::new(header_ptr + HEADER_SIZE)) } /// Deallocates the space which was allocated for a pointer. @@ -170,80 +365,83 @@ impl FreeingBumpHeapAllocator { /// /// - `mem` - a slice representing the linear memory on which this allocator operates. /// - `ptr` - pointer to the allocated chunk - pub fn deallocate(&mut self, mem: &mut [u8], ptr: Pointer) -> Result<(), Error> { - let ptr = u32::from(ptr) - self.ptr_offset; - let ptr = ptr.checked_sub(PREFIX_SIZE).ok_or_else(|| - error("Invalid pointer for deallocation") - )?; - - let list_index: usize = self.get_heap_u64(mem, ptr)? - .try_into() - .map_err(|_| error("read invalid list index"))?; - if list_index > self.heads.len() { - return Err(error("read invalid list index")); - } - self.set_heap_u64(mem, ptr, self.heads[list_index] as u64)?; - self.heads[list_index] = ptr; - - let item_size = Self::get_item_size_from_index(list_index); - self.total_size = self.total_size.checked_sub(item_size as u32 + PREFIX_SIZE) + pub fn deallocate(&mut self, mem: &mut M, ptr: Pointer) -> Result<(), Error> { + let header_ptr = u32::from(ptr) + .checked_sub(HEADER_SIZE) + .ok_or_else(|| error("Invalid pointer for deallocation"))?; + + let order = Header::read_from(mem, header_ptr)? + .into_occupied() + .ok_or_else(|| error("the allocation points to an empty header"))?; + + // Update the just freed header and knit it back to the free list. + let prev_head = self.free_lists.replace(order, Link::Ptr(header_ptr)); + Header::Free(prev_head).write_into(mem, header_ptr)?; + + // Do the total_size book keeping. + self.total_size = self + .total_size + .checked_sub(order.size() + HEADER_SIZE) .ok_or_else(|| error("Unable to subtract from total heap size without overflow"))?; trace!("Heap size is {} bytes after deallocation", self.total_size); Ok(()) } - /// Increases the `bumper` by `item_size + PREFIX_SIZE`. + /// Increases the `bumper` by `size`. /// /// Returns the `bumper` from before the increase. /// Returns an `Error::AllocatorOutOfSpace` if the operation /// would exhaust the heap. - fn bump(&mut self, item_size: u32, max_heap_size: u32) -> Result { - if self.bumper + PREFIX_SIZE + item_size > max_heap_size { + fn bump(&mut self, size: u32, heap_end: u32) -> Result { + if self.bumper + size > heap_end { return Err(Error::AllocatorOutOfSpace); } let res = self.bumper; - self.bumper += item_size + PREFIX_SIZE; + self.bumper += size; Ok(res) } +} - fn get_item_size_from_index(index: usize) -> usize { - // we shift 1 by three places, since the first possible item size is 8 - 1 << 3 << index - } +/// A trait for abstraction of accesses to linear memory. +pub trait Memory { + /// Read a u64 from the heap in LE form. Used to read heap allocation prefixes. + fn read_le_u64(&self, ptr: u32) -> Result; + /// Write a u64 to the heap in LE form. Used to write heap allocation prefixes. + fn write_le_u64(&mut self, ptr: u32, val: u64) -> Result<(), Error>; + /// Returns the full size of the memory. + fn size(&self) -> u32; +} - // Read a u64 from the heap in LE form. Used to read heap allocation prefixes. - fn get_heap_u64(&self, heap: &[u8], offset: u32) -> Result { - let range = self.heap_range(offset, 8, heap.len()) - .ok_or_else(|| error("read out of heap bounds"))?; - let bytes = heap[range].try_into() +impl Memory for [u8] { + fn read_le_u64(&self, ptr: u32) -> Result { + let range = + heap_range(ptr, 8, self.len()).ok_or_else(|| error("read out of heap bounds"))?; + let bytes = self[range] + .try_into() .expect("[u8] slice of length 8 must be convertible to [u8; 8]"); Ok(u64::from_le_bytes(bytes)) } - - // Write a u64 to the heap in LE form. Used to write heap allocation prefixes. - fn set_heap_u64(&self, heap: &mut [u8], offset: u32, val: u64) -> Result<(), Error> { - let range = self.heap_range(offset, 8, heap.len()) - .ok_or_else(|| error("write out of heap bounds"))?; + fn write_le_u64(&mut self, ptr: u32, val: u64) -> Result<(), Error> { + let range = + heap_range(ptr, 8, self.len()).ok_or_else(|| error("write out of heap bounds"))?; let bytes = val.to_le_bytes(); - &mut heap[range].copy_from_slice(&bytes[..]); + &mut self[range].copy_from_slice(&bytes[..]); Ok(()) } + fn size(&self) -> u32 { + u32::try_from(self.len()).expect("size of Wasm linear memory is <2^32; qed") + } +} - fn heap_range(&self, offset: u32, length: u32, heap_len: usize) -> Option> { - let start = offset - .checked_add(self.ptr_offset)? - as usize; - let end = offset - .checked_add(self.ptr_offset)? - .checked_add(length)? - as usize; - if end <= heap_len { - Some(start..end) - } else { - None - } +fn heap_range(offset: u32, length: u32, heap_len: usize) -> Option> { + let start = offset as usize; + let end = offset.checked_add(length)? as usize; + if end <= heap_len { + Some(start..end) + } else { + None } } @@ -268,8 +466,8 @@ mod tests { let ptr = heap.allocate(&mut mem[..], 1).unwrap(); // then - // returned pointer must start right after `PREFIX_SIZE` - assert_eq!(ptr, to_pointer(PREFIX_SIZE)); + // returned pointer must start right after `HEADER_SIZE` + assert_eq!(ptr, to_pointer(HEADER_SIZE)); } #[test] @@ -300,14 +498,14 @@ mod tests { // then // a prefix of 8 bytes is prepended to each pointer - assert_eq!(ptr1, to_pointer(PREFIX_SIZE)); + assert_eq!(ptr1, to_pointer(HEADER_SIZE)); // the prefix of 8 bytes + the content of ptr1 padded to the lowest possible // item size of 8 bytes + the prefix of ptr1 assert_eq!(ptr2, to_pointer(24)); // ptr2 + its content of 16 bytes + the prefix of 8 bytes - assert_eq!(ptr3, to_pointer(24 + 16 + PREFIX_SIZE)); + assert_eq!(ptr3, to_pointer(24 + 16 + HEADER_SIZE)); } #[test] @@ -317,7 +515,7 @@ mod tests { let mut heap = FreeingBumpHeapAllocator::new(0); let ptr1 = heap.allocate(&mut mem[..], 1).unwrap(); // the prefix of 8 bytes is prepended to the pointer - assert_eq!(ptr1, to_pointer(PREFIX_SIZE)); + assert_eq!(ptr1, to_pointer(HEADER_SIZE)); let ptr2 = heap.allocate(&mut mem[..], 1).unwrap(); // the prefix of 8 bytes + the content of ptr 1 is prepended to the pointer @@ -329,7 +527,7 @@ mod tests { // then // then the heads table should contain a pointer to the // prefix of ptr2 in the leftmost entry - assert_eq!(heap.heads[0], u32::from(ptr2) - PREFIX_SIZE); + assert_eq!(heap.free_lists.heads[0], Link::Ptr(u32::from(ptr2) - HEADER_SIZE)); } #[test] @@ -341,13 +539,13 @@ mod tests { let ptr1 = heap.allocate(&mut mem[..], 1).unwrap(); // the prefix of 8 bytes is prepended to the pointer - assert_eq!(ptr1, to_pointer(padded_offset + PREFIX_SIZE)); + assert_eq!(ptr1, to_pointer(padded_offset + HEADER_SIZE)); let ptr2 = heap.allocate(&mut mem[..], 9).unwrap(); // the padded_offset + the previously allocated ptr (8 bytes prefix + // 8 bytes content) + the prefix of 8 bytes which is prepended to the // current pointer - assert_eq!(ptr2, to_pointer(padded_offset + 16 + PREFIX_SIZE)); + assert_eq!(ptr2, to_pointer(padded_offset + 16 + HEADER_SIZE)); // when heap.deallocate(&mut mem[..], ptr2).unwrap(); @@ -355,8 +553,8 @@ mod tests { // then // should have re-allocated - assert_eq!(ptr3, to_pointer(padded_offset + 16 + PREFIX_SIZE)); - assert_eq!(heap.heads, [u32::max_value(); N]); + assert_eq!(ptr3, to_pointer(padded_offset + 16 + HEADER_SIZE)); + assert_eq!(heap.free_lists.heads, [Link::Null; N]); } #[test] @@ -375,12 +573,12 @@ mod tests { heap.deallocate(&mut mem[..], ptr3).unwrap(); // then - assert_eq!(heap.heads[0], u32::from(ptr3) - PREFIX_SIZE); + assert_eq!(heap.free_lists.heads[0], Link::Ptr(u32::from(ptr3) - HEADER_SIZE)); let ptr4 = heap.allocate(&mut mem[..], 8).unwrap(); assert_eq!(ptr4, ptr3); - assert_eq!(heap.heads[0], u32::from(ptr2) - PREFIX_SIZE); + assert_eq!(heap.free_lists.heads[0], Link::Ptr(u32::from(ptr2) - HEADER_SIZE)); } #[test] @@ -404,8 +602,8 @@ mod tests { // given let mut mem = [0u8; PAGE_SIZE as usize]; let mut heap = FreeingBumpHeapAllocator::new(0); - let ptr1 = heap.allocate(&mut mem[..], (PAGE_SIZE / 2) - PREFIX_SIZE).unwrap(); - assert_eq!(ptr1, to_pointer(PREFIX_SIZE)); + let ptr1 = heap.allocate(&mut mem[..], (PAGE_SIZE / 2) - HEADER_SIZE).unwrap(); + assert_eq!(ptr1, to_pointer(HEADER_SIZE)); // when let ptr2 = heap.allocate(&mut mem[..], PAGE_SIZE / 2); @@ -428,7 +626,7 @@ mod tests { let ptr = heap.allocate(&mut mem[..], MAX_POSSIBLE_ALLOCATION).unwrap(); // then - assert_eq!(ptr, to_pointer(PREFIX_SIZE)); + assert_eq!(ptr, to_pointer(HEADER_SIZE)); } #[test] @@ -454,7 +652,7 @@ mod tests { let mut heap = FreeingBumpHeapAllocator::new(0); let ptr1 = heap.allocate(&mut mem[..], 32).unwrap(); - assert_eq!(ptr1, to_pointer(PREFIX_SIZE)); + assert_eq!(ptr1, to_pointer(HEADER_SIZE)); heap.deallocate(&mut mem[..], ptr1).expect("failed freeing ptr1"); assert_eq!(heap.total_size, 0); assert_eq!(heap.bumper, 40); @@ -466,7 +664,7 @@ mod tests { assert_eq!(heap.bumper, 64); // when - // the `bumper` value is equal to `max_heap_size` here and any + // the `bumper` value is equal to `size` here and any // further allocation which would increment the bumper must fail. // we try to allocate 8 bytes here, which will increment the // bumper since no 8 byte item has been allocated+freed before. @@ -490,7 +688,7 @@ mod tests { heap.allocate(&mut mem[..], 9).unwrap(); // then - assert_eq!(heap.total_size, PREFIX_SIZE + 16); + assert_eq!(heap.total_size, HEADER_SIZE + 16); } #[test] @@ -501,7 +699,7 @@ mod tests { // when let ptr = heap.allocate(&mut mem[..], 42).unwrap(); - assert_eq!(ptr, to_pointer(16 + PREFIX_SIZE)); + assert_eq!(ptr, to_pointer(16 + HEADER_SIZE)); heap.deallocate(&mut mem[..], ptr).unwrap(); // then @@ -528,23 +726,22 @@ mod tests { fn should_read_and_write_u64_correctly() { // given let mut mem = [0u8; PAGE_SIZE as usize]; - let heap = FreeingBumpHeapAllocator::new(16); // when - heap.set_heap_u64(&mut mem[..], 40, 4480113).unwrap(); + Memory::write_le_u64(mem.as_mut(), 40, 4480113).unwrap(); // then - let value = heap.get_heap_u64(&mut mem[..], 40).unwrap(); + let value = Memory::read_le_u64(mem.as_mut(), 40).unwrap(); assert_eq!(value, 4480113); } #[test] - fn should_get_item_size_from_index() { + fn should_get_item_size_from_order() { // given - let index = 0; + let raw_order = 0; // when - let item_size = FreeingBumpHeapAllocator::get_item_size_from_index(index); + let item_size = Order::from_raw(raw_order).unwrap().size(); // then assert_eq!(item_size, 8); @@ -553,10 +750,10 @@ mod tests { #[test] fn should_get_max_item_size_from_index() { // given - let index = 21; + let raw_order = 21; // when - let item_size = FreeingBumpHeapAllocator::get_item_size_from_index(index); + let item_size = Order::from_raw(raw_order).unwrap().size(); // then assert_eq!(item_size as u32, MAX_POSSIBLE_ALLOCATION); @@ -568,10 +765,27 @@ mod tests { let mut heap = FreeingBumpHeapAllocator::new(0); // Allocate and free some pointers - let ptrs = (0..4).map(|_| heap.allocate(&mut mem, 8).unwrap()).collect::>(); - ptrs.into_iter().for_each(|ptr| heap.deallocate(&mut mem, ptr).unwrap()); + let ptrs = (0..4).map(|_| heap.allocate(&mut mem[..], 8).unwrap()).collect::>(); + ptrs.into_iter().for_each(|ptr| heap.deallocate(&mut mem[..], ptr).unwrap()); // Second time we should be able to allocate all of them again. - let _ = (0..4).map(|_| heap.allocate(&mut mem, 8).unwrap()).collect::>(); + let _ = (0..4).map(|_| heap.allocate(&mut mem[..], 8).unwrap()).collect::>(); + } + + #[test] + fn header_read_write() { + let roundtrip = |header: Header| { + let mut memory = [0u8; 32]; + header.write_into(memory.as_mut(), 0).unwrap(); + + let read_header = Header::read_from(memory.as_mut(), 0).unwrap(); + assert_eq!(header, read_header); + }; + + roundtrip(Header::Occupied(Order(0))); + roundtrip(Header::Occupied(Order(1))); + roundtrip(Header::Free(Link::Null)); + roundtrip(Header::Free(Link::Ptr(0))); + roundtrip(Header::Free(Link::Ptr(4))); } } diff --git a/primitives/application-crypto/src/lib.rs b/primitives/application-crypto/src/lib.rs index fed4a6bf8e..b7c9ccaa98 100644 --- a/primitives/application-crypto/src/lib.rs +++ b/primitives/application-crypto/src/lib.rs @@ -436,3 +436,30 @@ macro_rules! wrap { } } } + +/// Generate the given code if the pair type is available. +/// +/// The pair type is available when `feature = "std"` || `feature = "full_crypto"`. +/// +/// # Example +/// +/// ``` +/// sp_application_crypto::with_pair! { +/// pub type Pair = (); +/// } +/// ``` +#[macro_export] +#[cfg(any(feature = "std", feature = "full_crypto"))] +macro_rules! with_pair { + ( $( $def:tt )* ) => { + $( $def )* + } +} + + +#[doc(hidden)] +#[macro_export] +#[cfg(all(not(feature = "std"), not(feature = "full_crypto")))] +macro_rules! with_pair { + ( $( $def:tt )* ) => {} +} diff --git a/primitives/arithmetic/Cargo.toml b/primitives/arithmetic/Cargo.toml index 1b6d36ca37..995e36d5c9 100644 --- a/primitives/arithmetic/Cargo.toml +++ b/primitives/arithmetic/Cargo.toml @@ -14,7 +14,7 @@ serde = { version = "1.0.101", optional = true, features = ["derive"] } sp-debug-derive = { version = "2.0.0", default-features = false, path = "../../primitives/debug-derive" } [dev-dependencies] -primitive-types = "0.6.0" +primitive-types = "0.6.2" rand = "0.7.2" criterion = "0.3" diff --git a/primitives/arithmetic/fuzzer/Cargo.toml b/primitives/arithmetic/fuzzer/Cargo.toml index 56e789f3e4..19d677f744 100644 --- a/primitives/arithmetic/fuzzer/Cargo.toml +++ b/primitives/arithmetic/fuzzer/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0" [dependencies] sp-arithmetic = { version = "2.0.0", path = ".." } honggfuzz = "0.5" -primitive-types = "0.6" +primitive-types = "0.6.2" num-bigint = "0.2" num-traits = "0.2" diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index 0beef84758..c2feae00b7 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -40,5 +40,5 @@ mod fixed64; mod rational128; pub use fixed64::Fixed64; -pub use per_things::{Percent, Permill, Perbill, Perquintill}; +pub use per_things::{PerThing, Percent, Permill, Perbill, Perquintill}; pub use rational128::Rational128; diff --git a/primitives/arithmetic/src/per_things.rs b/primitives/arithmetic/src/per_things.rs index 554f3c3c21..cbb804baf5 100644 --- a/primitives/arithmetic/src/per_things.rs +++ b/primitives/arithmetic/src/per_things.rs @@ -19,50 +19,93 @@ use serde::{Serialize, Deserialize}; use sp_std::{ops, prelude::*, convert::TryInto}; use codec::{Encode, Decode, CompactAs}; -use crate::traits::{SaturatedConversion, UniqueSaturatedInto, Saturating}; +use crate::traits::{ + SaturatedConversion, UniqueSaturatedInto, Saturating, BaseArithmetic, +}; use sp_debug_derive::RuntimeDebug; +/// Something that implements a fixed point ration with an arbitrary granularity `X`, as _parts per +/// `X`_. +pub trait PerThing: Sized + Saturating + Copy { + /// The data type used to build this per-thingy. + type Inner: BaseArithmetic + Copy; + + /// accuracy of this type + const ACCURACY: Self::Inner; + + /// NoThing + fn zero() -> Self; + + /// `true` if this is nothing. + fn is_zero(&self) -> bool; + + /// Everything. + fn one() -> Self; + + /// Consume self and deconstruct into a raw numeric type. + fn deconstruct(self) -> Self::Inner; + + /// From an explicitly defined number of parts per maximum of the type. + fn from_parts(parts: Self::Inner) -> Self; + + /// Converts a percent into `Self`. Equal to `x / 100`. + fn from_percent(x: Self::Inner) -> Self; + + /// Return the product of multiplication of this value by itself. + fn square(self) -> Self; + + /// Converts a fraction into `Self`. + #[cfg(feature = "std")] + fn from_fraction(x: f64) -> Self; + + /// Approximate the fraction `p/q` into a per-thing fraction. This will never overflow. + /// + /// The computation of this approximation is performed in the generic type `N`. Given + /// `M` as the data type that can hold the maximum value of this per-thing (e.g. u32 for + /// perbill), this can only work if `N == M` or `N: From + TryInto`. + fn from_rational_approximation(p: N, q: N) -> Self + where N: Clone + Ord + From + TryInto + ops::Div; +} + macro_rules! implement_per_thing { ($name:ident, $test_mod:ident, [$($test_units:tt),+], $max:tt, $type:ty, $upper_type:ty, $title:expr $(,)?) => { /// A fixed point representation of a number between in the range [0, 1]. /// #[doc = $title] #[cfg_attr(feature = "std", derive(Serialize, Deserialize))] - #[derive(Encode, Decode, Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, CompactAs)] + #[derive(Encode, Decode, Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, RuntimeDebug, CompactAs)] pub struct $name($type); - impl $name { + impl PerThing for $name { + type Inner = $type; + + /// The accuracy of this type. + const ACCURACY: Self::Inner = $max; + /// Nothing. - pub fn zero() -> Self { Self(0) } + fn zero() -> Self { Self(0) } /// `true` if this is nothing. - pub fn is_zero(&self) -> bool { self.0 == 0 } + fn is_zero(&self) -> bool { self.0 == 0 } /// Everything. - pub fn one() -> Self { Self($max) } + fn one() -> Self { Self($max) } /// Consume self and deconstruct into a raw numeric type. - pub fn deconstruct(self) -> $type { self.0 } - - /// Return the scale at which this per-thing is working. - pub const fn accuracy() -> $type { $max } + fn deconstruct(self) -> Self::Inner { self.0 } /// From an explicitly defined number of parts per maximum of the type. - /// - /// This can be called at compile time. - pub const fn from_parts(parts: $type) -> Self { + fn from_parts(parts: Self::Inner) -> Self { Self([parts, $max][(parts > $max) as usize]) } /// Converts a percent into `Self`. Equal to `x / 100`. - /// - /// This can be created at compile time. - pub const fn from_percent(x: $type) -> Self { + fn from_percent(x: Self::Inner) -> Self { Self([x, 100][(x > 100) as usize] * ($max / 100)) } /// Return the product of multiplication of this value by itself. - pub fn square(self) -> Self { + fn square(self) -> Self { // both can be safely casted and multiplied. let p: $upper_type = self.0 as $upper_type * self.0 as $upper_type; let q: $upper_type = <$upper_type>::from($max) * <$upper_type>::from($max); @@ -71,33 +114,33 @@ macro_rules! implement_per_thing { /// Converts a fraction into `Self`. #[cfg(feature = "std")] - pub fn from_fraction(x: f64) -> Self { Self((x * ($max as f64)) as $type) } + fn from_fraction(x: f64) -> Self { Self((x * ($max as f64)) as Self::Inner) } /// Approximate the fraction `p/q` into a per-thing fraction. This will never overflow. /// /// The computation of this approximation is performed in the generic type `N`. Given /// `M` as the data type that can hold the maximum value of this per-thing (e.g. u32 for /// perbill), this can only work if `N == M` or `N: From + TryInto`. - pub fn from_rational_approximation(p: N, q: N) -> Self - where N: Clone + Ord + From<$type> + TryInto<$type> + ops::Div + fn from_rational_approximation(p: N, q: N) -> Self + where N: Clone + Ord + From + TryInto + ops::Div { // q cannot be zero. - let q = q.max((1 as $type).into()); + let q = q.max((1 as Self::Inner).into()); // p should not be bigger than q. let p = p.min(q.clone()); - let factor = (q.clone() / $max.into()).max((1 as $type).into()); + let factor = (q.clone() / $max.into()).max((1 as Self::Inner).into()); // q cannot overflow: (q / (q/$max)) < 2 * $max. p < q hence p also cannot overflow. - // this implies that $type must be able to fit 2 * $max. - let q_reduce: $type = (q / factor.clone()) + // this implies that Self::Inner must be able to fit 2 * $max. + let q_reduce: Self::Inner = (q / factor.clone()) .try_into() .map_err(|_| "Failed to convert") .expect( "q / (q/$max) < (2 * $max). Macro prevents any type being created that \ does not satisfy this; qed" ); - let p_reduce: $type = (p / factor.clone()) + let p_reduce: Self::Inner = (p / factor.clone()) .try_into() .map_err(|_| "Failed to convert") .expect( @@ -105,14 +148,39 @@ macro_rules! implement_per_thing { does not satisfy this; qed" ); - // `p_reduced` and `q_reduced` are withing $type. Mul by another $max will always - // fit in $upper_type. This is guaranteed by the macro tests. + // `p_reduced` and `q_reduced` are withing Self::Inner. Mul by another $max will + // always fit in $upper_type. This is guaranteed by the macro tests. let part = p_reduce as $upper_type * <$upper_type>::from($max) / q_reduce as $upper_type; - $name(part as $type) + $name(part as Self::Inner) + } + } + + /// Implement const functions + impl $name { + /// From an explicitly defined number of parts per maximum of the type. + /// + /// This can be called at compile time. + pub const fn from_parts(parts: $type) -> Self { + Self([parts, $max][(parts > $max) as usize]) + } + + /// Converts a percent into `Self`. Equal to `x / 100`. + /// + /// This can be created at compile time. + pub const fn from_percent(x: $type) -> Self { + Self([x, 100][(x > 100) as usize] * ($max / 100)) + } + + /// Everything. + /// + /// To avoid having to import `PerThing` when one needs to be used in test mocks. + #[cfg(feature = "std")] + pub fn one() -> Self { + ::one() } } @@ -190,7 +258,7 @@ macro_rules! implement_per_thing { #[cfg(test)] mod $test_mod { use codec::{Encode, Decode}; - use super::{$name, Saturating, RuntimeDebug}; + use super::{$name, Saturating, RuntimeDebug, PerThing}; use crate::traits::Zero; @@ -248,7 +316,7 @@ macro_rules! implement_per_thing { // some really basic stuff assert_eq!($name::zero(), $name::from_parts(Zero::zero())); assert_eq!($name::one(), $name::from_parts($max)); - assert_eq!($name::accuracy(), $max); + assert_eq!($name::ACCURACY, $max); assert_eq!($name::from_percent(0), $name::from_parts(Zero::zero())); assert_eq!($name::from_percent(10), $name::from_parts($max / 10)); assert_eq!($name::from_percent(100), $name::from_parts($max)); diff --git a/primitives/arithmetic/src/traits.rs b/primitives/arithmetic/src/traits.rs index ab525527e9..75adf0e136 100644 --- a/primitives/arithmetic/src/traits.rs +++ b/primitives/arithmetic/src/traits.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Primitives for the runtime modules. +//! Primitive traits for the runtime arithmetic. use sp_std::{self, convert::{TryFrom, TryInto}}; use codec::HasCompact; @@ -28,44 +28,55 @@ use sp_std::ops::{ RemAssign, Shl, Shr }; -/// A meta trait for arithmetic. -/// -/// Arithmetic types do all the usual stuff you'd expect numbers to do. They are guaranteed to -/// be able to represent at least `u32` values without loss, hence the trait implies `From` -/// and smaller ints. All other conversions are fallible. -pub trait SimpleArithmetic: +/// A meta trait for arithmetic type operations, regardless of any limitation on size. +pub trait BaseArithmetic: + From + Zero + One + IntegerSquareRoot + - From + From + From + TryInto + TryInto + TryInto + - TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + - UniqueSaturatedInto + UniqueSaturatedInto + UniqueSaturatedInto + - UniqueSaturatedFrom + UniqueSaturatedInto + UniqueSaturatedFrom + UniqueSaturatedInto + Add + AddAssign + Sub + SubAssign + Mul + MulAssign + Div + DivAssign + Rem + RemAssign + Shl + Shr + - CheckedShl + CheckedShr + CheckedAdd + CheckedSub + CheckedMul + CheckedDiv + - Saturating + PartialOrd + Ord + Bounded + - HasCompact + Sized + CheckedShl + CheckedShr + CheckedAdd + CheckedSub + CheckedMul + CheckedDiv + Saturating + + PartialOrd + Ord + Bounded + HasCompact + Sized + + TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + + TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto {} + impl + Zero + One + IntegerSquareRoot + - From + From + From + TryInto + TryInto + TryInto + - TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + - UniqueSaturatedInto + UniqueSaturatedInto + UniqueSaturatedInto + - UniqueSaturatedFrom + UniqueSaturatedInto + UniqueSaturatedFrom + - UniqueSaturatedInto + UniqueSaturatedFrom + UniqueSaturatedInto + Add + AddAssign + Sub + SubAssign + Mul + MulAssign + Div + DivAssign + Rem + RemAssign + Shl + Shr + - CheckedShl + CheckedShr + CheckedAdd + CheckedSub + CheckedMul + CheckedDiv + - Saturating + PartialOrd + Ord + Bounded + - HasCompact + Sized -> SimpleArithmetic for T {} + CheckedShl + CheckedShr + CheckedAdd + CheckedSub + CheckedMul + CheckedDiv + Saturating + + PartialOrd + Ord + Bounded + HasCompact + Sized + + TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + + TryFrom + TryInto + TryFrom + TryInto + TryFrom + TryInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto + + UniqueSaturatedFrom + UniqueSaturatedInto +> BaseArithmetic for T {} + +/// A meta trait for arithmetic. +/// +/// Arithmetic types do all the usual stuff you'd expect numbers to do. They are guaranteed to +/// be able to represent at least `u32` values without loss, hence the trait implies `From` +/// and smaller integers. All other conversions are fallible. +pub trait AtLeast32Bit: BaseArithmetic + From + From {} + +impl + From> AtLeast32Bit for T {} /// Just like `From` except that if the source value is too big to fit into the destination type /// then it'll saturate the destination. diff --git a/primitives/authority-discovery/src/lib.rs b/primitives/authority-discovery/src/lib.rs index fc76da61af..8926825525 100644 --- a/primitives/authority-discovery/src/lib.rs +++ b/primitives/authority-discovery/src/lib.rs @@ -25,9 +25,10 @@ mod app { app_crypto!(sr25519, AUTHORITY_DISCOVERY); } -/// An authority discovery authority keypair. -#[cfg(feature = "std")] -pub type AuthorityPair = app::Pair; +sp_application_crypto::with_pair! { + /// An authority discovery authority keypair. + pub type AuthorityPair = app::Pair; +} /// An authority discovery authority identifier. pub type AuthorityId = app::Public; diff --git a/primitives/blockchain/src/backend.rs b/primitives/blockchain/src/backend.rs index 101dcd1443..35cac1e481 100644 --- a/primitives/blockchain/src/backend.rs +++ b/primitives/blockchain/src/backend.rs @@ -59,19 +59,23 @@ pub trait HeaderBackend: Send + Sync { /// Get block header. Returns `UnknownBlock` error if block is not found. fn expect_header(&self, id: BlockId) -> Result { - self.header(id)?.ok_or_else(|| Error::UnknownBlock(format!("{}", id))) + self.header(id)?.ok_or_else(|| Error::UnknownBlock(format!("Expect header: {}", id))) } /// Convert an arbitrary block ID into a block number. Returns `UnknownBlock` error if block is not found. fn expect_block_number_from_id(&self, id: &BlockId) -> Result> { self.block_number_from_id(id) - .and_then(|n| n.ok_or_else(|| Error::UnknownBlock(format!("{}", id)))) + .and_then(|n| n.ok_or_else(|| + Error::UnknownBlock(format!("Expect block number from id: {}", id)) + )) } /// Convert an arbitrary block ID into a block hash. Returns `UnknownBlock` error if block is not found. fn expect_block_hash_from_id(&self, id: &BlockId) -> Result { self.block_hash_from_id(id) - .and_then(|n| n.ok_or_else(|| Error::UnknownBlock(format!("{}", id)))) + .and_then(|n| n.ok_or_else(|| + Error::UnknownBlock(format!("Expect block hash from id: {}", id)) + )) } } diff --git a/primitives/blockchain/src/header_metadata.rs b/primitives/blockchain/src/header_metadata.rs index fcd8062d1d..32dd0bcf06 100644 --- a/primitives/blockchain/src/header_metadata.rs +++ b/primitives/blockchain/src/header_metadata.rs @@ -151,7 +151,7 @@ pub fn tree_route>( } /// Hash and number of a block. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct HashAndNumber { /// The number of the block. pub number: NumberFor, diff --git a/primitives/consensus/aura/src/lib.rs b/primitives/consensus/aura/src/lib.rs index 4f20456667..2dda5b28bf 100644 --- a/primitives/consensus/aura/src/lib.rs +++ b/primitives/consensus/aura/src/lib.rs @@ -30,9 +30,10 @@ pub mod sr25519 { app_crypto!(sr25519, AURA); } - /// An Aura authority keypair using S/R 25519 as its crypto. - #[cfg(feature = "std")] - pub type AuthorityPair = app_sr25519::Pair; + sp_application_crypto::with_pair! { + /// An Aura authority keypair using S/R 25519 as its crypto. + pub type AuthorityPair = app_sr25519::Pair; + } /// An Aura authority signature using S/R 25519 as its crypto. pub type AuthoritySignature = app_sr25519::Signature; @@ -47,9 +48,10 @@ pub mod ed25519 { app_crypto!(ed25519, AURA); } - /// An Aura authority keypair using Ed25519 as its crypto. - #[cfg(feature = "std")] - pub type AuthorityPair = app_ed25519::Pair; + sp_application_crypto::with_pair! { + /// An Aura authority keypair using Ed25519 as its crypto. + pub type AuthorityPair = app_ed25519::Pair; + } /// An Aura authority signature using Ed25519 as its crypto. pub type AuthoritySignature = app_ed25519::Signature; diff --git a/primitives/consensus/common/Cargo.toml b/primitives/consensus/common/Cargo.toml index 1b5c54b7e9..2abc6f5617 100644 --- a/primitives/consensus/common/Cargo.toml +++ b/primitives/consensus/common/Cargo.toml @@ -14,7 +14,7 @@ sp-core = { path= "../../core" } sp-inherents = { version = "2.0.0", path = "../../inherents" } sp-state-machine = { version = "0.8.0", path = "../../../primitives/state-machine" } futures = { version = "0.3.1", features = ["thread-pool"] } -futures-timer = "0.4.0" +futures-timer = "3.0.1" futures-diagnose = "1.0" sp-std = { version = "2.0.0", path = "../../std" } sp-version = { version = "2.0.0", path = "../../version" } diff --git a/primitives/core/Cargo.toml b/primitives/core/Cargo.toml index 6f1ce2f080..574b9aa9cb 100644 --- a/primitives/core/Cargo.toml +++ b/primitives/core/Cargo.toml @@ -12,7 +12,7 @@ rustc-hex = { version = "2.0.1", default-features = false } log = { version = "0.4.8", default-features = false } serde = { version = "1.0.101", optional = true, features = ["derive"] } byteorder = { version = "1.3.2", default-features = false } -primitive-types = { version = "0.6", default-features = false, features = ["codec"] } +primitive-types = { version = "0.6.2", default-features = false, features = ["codec"] } impl-serde = { version = "0.2.3", optional = true } wasmi = { version = "0.6.2", optional = true } hash-db = { version = "0.15.2", default-features = false } diff --git a/primitives/core/benches/bench.rs b/primitives/core/benches/bench.rs index 7096cd6c19..7db9d72e6b 100644 --- a/primitives/core/benches/bench.rs +++ b/primitives/core/benches/bench.rs @@ -86,9 +86,29 @@ fn bench_ed25519(c: &mut Criterion) { }, vec![32, 1024, 1024 * 1024]); } +fn bench_sr25519(c: &mut Criterion) { + c.bench_function_over_inputs("signing - sr25519", |b, &msg_size| { + let msg = (0..msg_size) + .map(|_| rand::random::()) + .collect::>(); + let key = sp_core::sr25519::Pair::generate().0; + b.iter(|| key.sign(&msg)) + }, vec![32, 1024, 1024 * 1024]); + + c.bench_function_over_inputs("verifying - sr25519", |b, &msg_size| { + let msg = (0..msg_size) + .map(|_| rand::random::()) + .collect::>(); + let key = sp_core::sr25519::Pair::generate().0; + let sig = key.sign(&msg); + let public = key.public(); + b.iter(|| sp_core::sr25519::Pair::verify(&sig, &msg, &public)) + }, vec![32, 1024, 1024 * 1024]); +} + criterion_group!{ name = benches; config = Criterion::default().warm_up_time(Duration::from_millis(500)).without_plots(); - targets = bench_hash_128_fix_size, bench_hash_128_dyn_size, bench_ed25519 + targets = bench_hash_128_fix_size, bench_hash_128_dyn_size, bench_ed25519, bench_sr25519 } criterion_main!(benches); diff --git a/primitives/core/src/lib.rs b/primitives/core/src/lib.rs index 5bb9a3927f..b1a5916c92 100644 --- a/primitives/core/src/lib.rs +++ b/primitives/core/src/lib.rs @@ -70,9 +70,6 @@ mod changes_trie; pub mod traits; pub mod testing; -#[cfg(test)] -mod tests; - pub use self::hash::{H160, H256, H512, convert_hash}; pub use self::uint::U256; pub use changes_trie::{ChangesTrieConfiguration, ChangesTrieConfigurationRange}; diff --git a/primitives/core/src/tests.rs b/primitives/core/src/tests.rs deleted file mode 100644 index 1eda2157c4..0000000000 --- a/primitives/core/src/tests.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017-2020 Parity Technologies (UK) Ltd. -// This file is part of Substrate. - -// Substrate is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Substrate is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Substrate. If not, see . - -//! Tests. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 350b65d190..75193a4b9c 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -202,6 +202,14 @@ pub trait Externalities: ExtensionStore { /// /// Returns the SCALE encoded hash. fn storage_changes_root(&mut self, parent: &[u8]) -> Result>, ()>; + + fn wipe(&mut self) { + unimplemented!() + } + + fn commit(&mut self) { + unimplemented!() + } } /// Extension for the [`Externalities`] trait. diff --git a/primitives/finality-grandpa/Cargo.toml b/primitives/finality-grandpa/Cargo.toml index dfa6b572d8..6356ec82e5 100644 --- a/primitives/finality-grandpa/Cargo.toml +++ b/primitives/finality-grandpa/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" license = "GPL-3.0" [dependencies] -app-crypto = { version = "2.0.0", default-features = false, package = "sp-application-crypto", path = "../application-crypto" } +sp-application-crypto = { version = "2.0.0", default-features = false, path = "../application-crypto" } codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } sp-std = { version = "2.0.0", default-features = false, path = "../std" } serde = { version = "1.0.101", optional = true, features = ["derive"] } @@ -16,7 +16,7 @@ sp-runtime = { version = "2.0.0", default-features = false, path = "../runtime" [features] default = ["std"] std = [ - "app-crypto/std", + "sp-application-crypto/std", "codec/std", "sp-std/std", "serde", diff --git a/primitives/finality-grandpa/src/lib.rs b/primitives/finality-grandpa/src/lib.rs index f1481c0aed..205fce0ce3 100644 --- a/primitives/finality-grandpa/src/lib.rs +++ b/primitives/finality-grandpa/src/lib.rs @@ -29,13 +29,14 @@ use sp_std::borrow::Cow; use sp_std::vec::Vec; mod app { - use app_crypto::{app_crypto, key_types::GRANDPA, ed25519}; + use sp_application_crypto::{app_crypto, key_types::GRANDPA, ed25519}; app_crypto!(ed25519, GRANDPA); } -/// The grandpa crypto scheme defined via the keypair type. -#[cfg(feature = "std")] -pub type AuthorityPair = app::Pair; +sp_application_crypto::with_pair! { + /// The grandpa crypto scheme defined via the keypair type. + pub type AuthorityPair = app::Pair; +} /// Identity of a Grandpa authority. pub type AuthorityId = app::Public; diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 1b531725fe..ce8a546e86 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -771,6 +771,30 @@ pub trait Logging { } } +/// Interface that provides functions for benchmarking the runtime. +#[runtime_interface] +pub trait Benchmarking { + /// Get the number of nanoseconds passed since the UNIX epoch + /// + /// WARNING! This is a non-deterministic call. Do not use this within + /// consensus critical logic. + fn current_time() -> u128 { + std::time::SystemTime::now().duration_since(std::time::SystemTime::UNIX_EPOCH) + .expect("Unix time doesn't go backwards; qed") + .as_nanos() + } + + /// Reset the trie database to the genesis state. + fn wipe_db(&mut self) { + self.wipe() + } + + /// Commit pending storage changes to the trie database and clear the database cache. + fn commit_db(&mut self) { + self.commit() + } +} + /// Wasm-only interface that provides functions for interacting with the sandbox. #[runtime_interface(wasm_only)] pub trait Sandbox { diff --git a/primitives/keyring/src/sr25519.rs b/primitives/keyring/src/sr25519.rs index d3776f4073..476997f2db 100644 --- a/primitives/keyring/src/sr25519.rs +++ b/primitives/keyring/src/sr25519.rs @@ -86,7 +86,6 @@ impl Keyring { pub fn public(self) -> Public { self.pair().public() } - pub fn to_seed(self) -> String { format!("//{}", self) } diff --git a/primitives/phragmen/src/lib.rs b/primitives/phragmen/src/lib.rs index c13654543d..e2f77e5458 100644 --- a/primitives/phragmen/src/lib.rs +++ b/primitives/phragmen/src/lib.rs @@ -33,10 +33,14 @@ #![cfg_attr(not(feature = "std"), no_std)] -use sp_std::{prelude::*, collections::btree_map::BTreeMap}; -use sp_runtime::RuntimeDebug; -use sp_runtime::{helpers_128bit::multiply_by_rational, Perbill, Rational128}; -use sp_runtime::traits::{Zero, Convert, Member, SimpleArithmetic, Saturating, Bounded}; +use sp_std::{prelude::*, collections::btree_map::BTreeMap, convert::TryFrom}; +use sp_runtime::{ + PerThing, Rational128, RuntimeDebug, + helpers_128bit::multiply_by_rational, +}; +use sp_runtime::traits::{ + Zero, Convert, Member, AtLeast32Bit, SaturatedConversion, Bounded, Saturating, +}; #[cfg(test)] mod mock; @@ -93,21 +97,21 @@ pub struct Edge { candidate_index: usize, } -/// Means a particular `AccountId` was backed by `Perbill`th of a nominator's stake. -pub type PhragmenAssignment = (AccountId, Perbill); +/// Particular `AccountId` was backed by `T`-ratio of a nominator's stake. +pub type PhragmenAssignment = (AccountId, T); -/// Means a particular `AccountId` was backed by `ExtendedBalance` of a nominator's stake. +/// Particular `AccountId` was backed by `ExtendedBalance` of a nominator's stake. pub type PhragmenStakedAssignment = (AccountId, ExtendedBalance); /// Final result of the phragmen election. #[derive(RuntimeDebug)] -pub struct PhragmenResult { +pub struct PhragmenResult { /// Just winners zipped with their approval stake. Note that the approval stake is merely the /// sub of their received stake and could be used for very basic sorting and approval voting. pub winners: Vec<(AccountId, ExtendedBalance)>, /// Individual assignments. for each tuple, the first elements is a voter and the second /// is the list of candidates that it supports. - pub assignments: Vec<(AccountId, Vec>)> + pub assignments: Vec<(AccountId, Vec>)> } /// A structure to demonstrate the phragmen result from the perspective of the candidate, i.e. how @@ -145,23 +149,24 @@ pub type SupportMap = BTreeMap>; /// responsibility of the caller to make sure only those candidates who have a sensible economic /// value are passed in. From the perspective of this function, a candidate can easily be among the /// winner with no backing stake. -pub fn elect( +pub fn elect( candidate_count: usize, minimum_candidate_count: usize, initial_candidates: Vec, initial_voters: Vec<(AccountId, Vec)>, stake_of: FS, -) -> Option> where +) -> Option> where AccountId: Default + Ord + Member, - Balance: Default + Copy + SimpleArithmetic, + Balance: Default + Copy + AtLeast32Bit, for<'r> FS: Fn(&'r AccountId) -> Balance, C: Convert + Convert, + R: PerThing, { let to_votes = |b: Balance| >::convert(b) as ExtendedBalance; // return structures let mut elected_candidates: Vec<(AccountId, ExtendedBalance)>; - let mut assigned: Vec<(AccountId, Vec>)>; + let mut assigned: Vec<(AccountId, Vec>)>; // used to cache and access candidates index. let mut c_idx_cache = BTreeMap::::new(); @@ -272,20 +277,29 @@ pub fn elect( let mut assignment = (n.who.clone(), vec![]); for e in &mut n.edges { if elected_candidates.iter().position(|(ref c, _)| *c == e.who).is_some() { - let per_bill_parts = + let per_bill_parts: R::Inner = { if n.load == e.load { // Full support. No need to calculate. - Perbill::accuracy().into() + R::ACCURACY } else { if e.load.d() == n.load.d() { // return e.load / n.load. - let desired_scale: u128 = Perbill::accuracy().into(); - multiply_by_rational( + let desired_scale: u128 = R::ACCURACY.saturated_into(); + let parts = multiply_by_rational( desired_scale, e.load.n(), n.load.n(), - ).unwrap_or(Bounded::max_value()) + ) + // If result cannot fit in u128. Not much we can do about it. + .unwrap_or(Bounded::max_value()); + + TryFrom::try_from(parts) + // If the result cannot fit into R::Inner. Defensive only. This can + // never happen. `desired_scale * e / n`, where `e / n < 1` always + // yields a value smaller than `desired_scale`, which will fit into + // R::Inner. + .unwrap_or(Bounded::max_value()) } else { // defensive only. Both edge and nominator loads are built from // scores, hence MUST have the same denominator. @@ -293,10 +307,7 @@ pub fn elect( } } }; - // safer to .min() inside as well to argue as u32 is safe. - let per_thing = Perbill::from_parts( - per_bill_parts.min(Perbill::accuracy().into()) as u32 - ); + let per_thing = R::from_parts(per_bill_parts); assignment.1.push((e.who.clone(), per_thing)); } } @@ -304,20 +315,19 @@ pub fn elect( if assignment.1.len() > 0 { // To ensure an assertion indicating: no stake from the nominator going to waste, // we add a minimal post-processing to equally assign all of the leftover stake ratios. - let vote_count = assignment.1.len() as u32; + let vote_count: R::Inner = assignment.1.len().saturated_into(); let len = assignment.1.len(); - let sum = assignment.1.iter() - .map(|a| a.1.deconstruct()) - .sum::(); - let accuracy = Perbill::accuracy(); - let diff = accuracy.checked_sub(sum).unwrap_or(0); + let mut sum: R::Inner = Zero::zero(); + assignment.1.iter().for_each(|a| sum = sum.saturating_add(a.1.deconstruct())); + let accuracy = R::ACCURACY; + let diff = accuracy.saturating_sub(sum); let diff_per_vote = (diff / vote_count).min(accuracy); - if diff_per_vote > 0 { + if !diff_per_vote.is_zero() { for i in 0..len { let current_ratio = assignment.1[i % len].1; let next_ratio = current_ratio - .saturating_add(Perbill::from_parts(diff_per_vote)); + .saturating_add(R::from_parts(diff_per_vote)); assignment.1[i % len].1 = next_ratio; } } @@ -325,9 +335,9 @@ pub fn elect( // `remainder` is set to be less than maximum votes of a nominator (currently 16). // safe to cast it to usize. let remainder = diff - diff_per_vote * vote_count; - for i in 0..remainder as usize { + for i in 0..remainder.saturated_into::() { let current_ratio = assignment.1[i % len].1; - let next_ratio = current_ratio.saturating_add(Perbill::from_parts(1)); + let next_ratio = current_ratio.saturating_add(R::from_parts(1u8.into())); assignment.1[i % len].1 = next_ratio; } assigned.push(assignment); @@ -341,15 +351,16 @@ pub fn elect( } /// Build the support map from the given phragmen result. -pub fn build_support_map( +pub fn build_support_map( elected_stashes: &Vec, - assignments: &Vec<(AccountId, Vec>)>, + assignments: &Vec<(AccountId, Vec>)>, stake_of: FS, ) -> SupportMap where AccountId: Default + Ord + Member, - Balance: Default + Copy + SimpleArithmetic, + Balance: Default + Copy + AtLeast32Bit, C: Convert + Convert, for<'r> FS: Fn(&'r AccountId) -> Balance, + R: PerThing + sp_std::ops::Mul, { let to_votes = |b: Balance| >::convert(b) as ExtendedBalance; // Initialize the support of each candidate. diff --git a/primitives/phragmen/src/mock.rs b/primitives/phragmen/src/mock.rs index 59b8d4eb5c..b3110a5dba 100644 --- a/primitives/phragmen/src/mock.rs +++ b/primitives/phragmen/src/mock.rs @@ -20,7 +20,7 @@ use crate::{elect, PhragmenResult, PhragmenAssignment}; use sp_runtime::{ - assert_eq_error_rate, Perbill, + assert_eq_error_rate, Perbill, PerThing, traits::{Convert, Member, SaturatedConversion} }; use sp_std::collections::btree_map::BTreeMap; @@ -320,10 +320,10 @@ pub(crate) fn create_stake_of(stakes: &[(AccountId, Balance)]) } -pub fn check_assignments(assignments: Vec<(AccountId, Vec>)>) { +pub fn check_assignments(assignments: Vec<(AccountId, Vec>)>) { for (_, a) in assignments { let sum: u32 = a.iter().map(|(_, p)| p.deconstruct()).sum(); - assert_eq_error_rate!(sum, Perbill::accuracy(), 5); + assert_eq_error_rate!(sum, Perbill::ACCURACY, 5); } } @@ -335,7 +335,7 @@ pub(crate) fn run_and_compare( min_to_elect: usize, ) { // run fixed point code. - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Perbill>( to_elect, min_to_elect, candidates.clone(), diff --git a/primitives/phragmen/src/tests.rs b/primitives/phragmen/src/tests.rs index b182cfca05..9027cc335f 100644 --- a/primitives/phragmen/src/tests.rs +++ b/primitives/phragmen/src/tests.rs @@ -23,6 +23,8 @@ use crate::{elect, PhragmenResult, PhragmenStakedAssignment, build_support_map, use substrate_test_utils::assert_eq_uvec; use sp_runtime::Perbill; +type Output = Perbill; + #[test] fn float_phragmen_poc_works() { let candidates = vec![1, 2, 3]; @@ -78,7 +80,7 @@ fn phragmen_poc_works() { (30, vec![2, 3]), ]; - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates, @@ -147,7 +149,7 @@ fn phragmen_accuracy_on_large_scale_only_validators() { (5, (u64::max_value() - 2).into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates.clone(), @@ -178,7 +180,7 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { (14, u64::max_value().into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates, @@ -210,7 +212,7 @@ fn phragmen_accuracy_on_small_scale_self_vote() { (30, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote, Output>( 3, 3, candidates, @@ -241,7 +243,7 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote, Output>( 3, 3, candidates, @@ -275,7 +277,7 @@ fn phragmen_large_scale_test() { (50, 990000000000000000), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates, @@ -302,7 +304,7 @@ fn phragmen_large_scale_test_2() { (50, nom_budget.into()), ]); - let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments } = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates, @@ -367,7 +369,7 @@ fn elect_has_no_entry_barrier() { (2, 10), ]); - let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote>( + let PhragmenResult { winners, assignments: _ } = elect::<_, _, _, TestCurrencyToVote, Output>( 3, 3, candidates, @@ -395,7 +397,7 @@ fn minimum_to_elect_is_respected() { (2, 10), ]); - let maybe_result = elect::<_, _, _, TestCurrencyToVote>( + let maybe_result = elect::<_, _, _, TestCurrencyToVote, Output>( 10, 10, candidates, @@ -422,7 +424,7 @@ fn self_votes_should_be_kept() { (1, 8), ]); - let result = elect::<_, _, _, TestCurrencyToVote>( + let result = elect::<_, _, _, TestCurrencyToVote, Output>( 2, 2, candidates, @@ -448,7 +450,8 @@ fn self_votes_should_be_kept() { Balance, AccountId, _, - TestCurrencyToVote + TestCurrencyToVote, + Output, >( &result.winners.into_iter().map(|(who, _)| who).collect(), &result.assignments, diff --git a/primitives/runtime-interface/Cargo.toml b/primitives/runtime-interface/Cargo.toml index 3f20de1288..cef3acdbfb 100644 --- a/primitives/runtime-interface/Cargo.toml +++ b/primitives/runtime-interface/Cargo.toml @@ -12,7 +12,7 @@ sp-runtime-interface-proc-macro = { version = "2.0.0", path = "proc-macro" } sp-externalities = { version = "0.8.0", optional = true, path = "../externalities" } codec = { package = "parity-scale-codec", version = "1.0.6", default-features = false } static_assertions = "1.0.0" -primitive-types = { version = "0.6.1", default-features = false } +primitive-types = { version = "0.6.2", default-features = false } [dev-dependencies] sp-runtime-interface-test-wasm = { version = "2.0.0", path = "test-wasm" } diff --git a/primitives/runtime-interface/test-wasm/src/lib.rs b/primitives/runtime-interface/test-wasm/src/lib.rs index c6e2c9909f..467f58cb30 100644 --- a/primitives/runtime-interface/test-wasm/src/lib.rs +++ b/primitives/runtime-interface/test-wasm/src/lib.rs @@ -108,6 +108,7 @@ pub trait TestApi { /// Two random external functions from the old runtime interface. /// This ensures that we still inherently export these functions from the host and that we are still /// compatible with old wasm runtimes. +#[cfg(not(feature = "std"))] extern "C" { pub fn ext_clear_storage(key_data: *const u8, key_len: u32); pub fn ext_keccak_256(data: *const u8, len: u32, out: *mut u8); @@ -115,6 +116,7 @@ extern "C" { /// Make sure the old runtime interface needs to be imported. #[no_mangle] +#[cfg(not(feature = "std"))] pub fn force_old_runtime_interface_import() { unsafe { ext_clear_storage(sp_std::ptr::null(), 0); } unsafe { ext_keccak_256(sp_std::ptr::null(), 0, sp_std::ptr::null_mut()); } diff --git a/primitives/runtime/src/curve.rs b/primitives/runtime/src/curve.rs index b45501fdb7..a230e2f32d 100644 --- a/primitives/runtime/src/curve.rs +++ b/primitives/runtime/src/curve.rs @@ -16,7 +16,7 @@ //! Provides some utilities to define a piecewise linear function. -use crate::{Perbill, traits::{SimpleArithmetic, SaturatedConversion}}; +use crate::{Perbill, PerThing, traits::{AtLeast32Bit, SaturatedConversion}}; use core::ops::Sub; /// Piecewise Linear function in [0, 1] -> [0, 1]. @@ -35,7 +35,7 @@ fn abs_sub + Clone>(a: N, b: N) -> N where { impl<'a> PiecewiseLinear<'a> { /// Compute `f(n/d)*d` with `n <= d`. This is useful to avoid loss of precision. pub fn calculate_for_fraction_times_denominator(&self, n: N, d: N) -> N where - N: SimpleArithmetic + Clone + N: AtLeast32Bit + Clone { let n = n.min(d.clone()); @@ -79,7 +79,7 @@ impl<'a> PiecewiseLinear<'a> { // This is guaranteed not to overflow on whatever values nor lose precision. // `q` must be superior to zero. fn multiply_by_rational_saturating(value: N, p: u32, q: u32) -> N - where N: SimpleArithmetic + Clone + where N: AtLeast32Bit + Clone { let q = q.max(1); diff --git a/primitives/runtime/src/generic/digest.rs b/primitives/runtime/src/generic/digest.rs index 4d09b58793..9d4fbe48d5 100644 --- a/primitives/runtime/src/generic/digest.rs +++ b/primitives/runtime/src/generic/digest.rs @@ -87,6 +87,12 @@ pub enum DigestItem { /// the consensus engine can (and should) read them itself to avoid /// code and state duplication. It is erroneous for a runtime to produce /// these, but this is not (yet) checked. + /// + /// NOTE: the runtime is not allowed to panic or fail in an `on_initialize` + /// call if an expected `PreRuntime` digest is not present. It is the + /// responsibility of a external block verifier to check this. Runtime API calls + /// will initialize the block without pre-runtime digests, so initialization + /// cannot fail when they are missing. PreRuntime(ConsensusEngineId, Vec), /// A message from the runtime to the consensus engine. This should *never* diff --git a/primitives/runtime/src/generic/header.rs b/primitives/runtime/src/generic/header.rs index 5bc7932feb..5efb36603d 100644 --- a/primitives/runtime/src/generic/header.rs +++ b/primitives/runtime/src/generic/header.rs @@ -20,7 +20,7 @@ use serde::{Deserialize, Serialize}; use crate::codec::{Decode, Encode, Codec, Input, Output, HasCompact, EncodeAsRef, Error}; use crate::traits::{ - self, Member, SimpleArithmetic, SimpleBitOps, Hash as HashT, + self, Member, AtLeast32Bit, SimpleBitOps, Hash as HashT, MaybeSerializeDeserialize, MaybeSerialize, MaybeDisplay, MaybeMallocSizeOf, }; @@ -122,7 +122,7 @@ impl codec::EncodeLike for Header where impl traits::Header for Header where Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + MaybeDisplay + - SimpleArithmetic + Codec + Copy + Into + TryFrom + sp_std::str::FromStr + + AtLeast32Bit + Codec + Copy + Into + TryFrom + sp_std::str::FromStr + MaybeMallocSizeOf, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash + Copy + Member + Ord + @@ -170,7 +170,7 @@ impl traits::Header for Header where } impl Header where - Number: Member + sp_std::hash::Hash + Copy + MaybeDisplay + SimpleArithmetic + Codec + Into + TryFrom, + Number: Member + sp_std::hash::Hash + Copy + MaybeDisplay + AtLeast32Bit + Codec + Into + TryFrom, Hash: HashT, Hash::Output: Default + sp_std::hash::Hash + Copy + Member + MaybeDisplay + SimpleBitOps + Codec, { diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index 44b0e417f3..5f6e210954 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -68,7 +68,7 @@ pub use sp_application_crypto::{RuntimeAppPublic, BoundToRuntimeAppPublic}; pub use sp_core::RuntimeDebug; /// Re-export top-level arithmetic stuff. -pub use sp_arithmetic::{Perquintill, Perbill, Permill, Percent, Rational128, Fixed64}; +pub use sp_arithmetic::{Perquintill, Perbill, Permill, Percent, Rational128, Fixed64, PerThing}; /// Re-export 128 bit helpers. pub use sp_arithmetic::helpers_128bit; /// Re-export big_uint stuff. @@ -689,6 +689,18 @@ pub fn print(print: impl traits::Printable) { print.print(); } +/// An alphabet of possible parameters to use for benchmarking. +#[derive(Encode, Decode, Clone, Copy, PartialEq, Debug)] +#[allow(missing_docs)] +pub enum BenchmarkParameter { + A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V, W, X, Y, Z, +} + +/// Results from running benchmarks on a FRAME pallet. +/// Contains duration of the function call in nanoseconds along with the benchmark parameters +/// used for that benchmark result. +pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128); + #[cfg(test)] mod tests { use super::*; diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index 82f8b7c6af..cf6facc46d 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -22,15 +22,18 @@ use sp_io; #[cfg(feature = "std")] use std::fmt::Display; #[cfg(feature = "std")] +use std::str::FromStr; +#[cfg(feature = "std")] use serde::{Serialize, Deserialize, de::DeserializeOwned}; use sp_core::{self, Hasher, Blake2Hasher, TypeId, RuntimeDebug}; +use crate::BenchmarkParameter; use crate::codec::{Codec, Encode, Decode}; use crate::transaction_validity::{ ValidTransaction, TransactionValidity, TransactionValidityError, UnknownTransaction, }; use crate::generic::{Digest, DigestItem}; pub use sp_arithmetic::traits::{ - SimpleArithmetic, UniqueSaturatedInto, UniqueSaturatedFrom, Saturating, SaturatedConversion, + AtLeast32Bit, UniqueSaturatedInto, UniqueSaturatedFrom, Saturating, SaturatedConversion, Zero, One, Bounded, CheckedAdd, CheckedSub, CheckedMul, CheckedDiv, CheckedShl, CheckedShr, IntegerSquareRoot }; @@ -554,6 +557,9 @@ sp_core::impl_maybe_marker!( /// A type that implements Display when in std environment. trait MaybeDisplay: Display; + /// A type that implements FromStr when in std environment. + trait MaybeFromStr: FromStr; + /// A type that implements Hash when in std environment. trait MaybeHash: sp_std::hash::Hash; @@ -567,22 +573,6 @@ sp_core::impl_maybe_marker!( trait MaybeMallocSizeOf: parity_util_mem::MallocSizeOf; ); -/// A type that provides a randomness beacon. -pub trait RandomnessBeacon { - /// Returns 32 bytes of random data. The output will change eventually, but - /// is not guaranteed to be different between any two calls. - /// - /// # Security - /// - /// This MUST NOT be used for gambling, as it can be influenced by a - /// malicious validator in the short term. It MAY be used in many - /// cryptographic protocols, however, so long as one remembers that this - /// (like everything else on-chain) is public. For example, it can be - /// used where a number is needed that cannot have been chosen by an - /// adversary, for purposes such as public-coin zero-knowledge proofs. - fn random() -> [u8; 32]; -} - /// A type that can be used in runtime structures. pub trait Member: Send + Sync + Sized + Debug + Eq + PartialEq + Clone + 'static {} impl Member for T {} @@ -604,7 +594,7 @@ pub trait Header: { /// Header number. type Number: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash - + Copy + MaybeDisplay + SimpleArithmetic + Codec + sp_std::str::FromStr + + Copy + MaybeDisplay + AtLeast32Bit + Codec + sp_std::str::FromStr + MaybeMallocSizeOf; /// Header hash type type Hash: Member + MaybeSerializeDeserialize + Debug + sp_std::hash::Hash + Ord @@ -1486,6 +1476,75 @@ pub trait BlockIdTo { ) -> Result>, Self::Error>; } +/// The pallet benchmarking trait. +pub trait Benchmarking { + /// Run the benchmarks for this pallet. + /// + /// Parameters + /// - `extrinsic`: The name of extrinsic function you want to benchmark encoded as bytes. + /// - `steps`: The number of sample points you want to take across the range of parameters. + /// - `repeat`: The number of times you want to repeat a benchmark. + fn run_benchmark(extrinsic: Vec, steps: u32, repeat: u32) -> Result, &'static str>; +} + +/// The required setup for creating a benchmark. +pub trait BenchmarkingSetup { + /// Return the components and their ranges which should be tested in this benchmark. + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)>; + + /// Set up the storage, and prepare a call and caller to test in a single run of the benchmark. + fn instance(&self, components: &[(BenchmarkParameter, u32)]) -> Result<(Call, RawOrigin), &'static str>; +} + +/// Creates a `SelectedBenchmark` enum implementing `BenchmarkingSetup`. +/// +/// Every variant must implement [`BenchmarkingSetup`](crate::traits::BenchmarkingSetup). +/// +/// ```nocompile +/// +/// struct Transfer; +/// impl BenchmarkingSetup for Transfer { ... } +/// +/// struct SetBalance; +/// impl BenchmarkingSetup for SetBalance { ... } +/// +/// selected_benchmark!(Transfer, SetBalance); +/// ``` +#[macro_export] +macro_rules! selected_benchmark { + ($($bench:ident),*) => { + // The list of available benchmarks for this pallet. + enum SelectedBenchmark { + $( $bench, )* + } + + // Allow us to select a benchmark from the list of available benchmarks. + impl $crate::traits::BenchmarkingSetup, RawOrigin> for SelectedBenchmark { + fn components(&self) -> Vec<(BenchmarkParameter, u32, u32)> { + match self { + $( Self::$bench => <$bench as $crate::traits::BenchmarkingSetup< + T, + Call, + RawOrigin, + >>::components(&$bench), )* + } + } + + fn instance(&self, components: &[(BenchmarkParameter, u32)]) + -> Result<(Call, RawOrigin), &'static str> + { + match self { + $( Self::$bench => <$bench as $crate::traits::BenchmarkingSetup< + T, + Call, + RawOrigin, + >>::instance(&$bench, components), )* + } + } + } + }; +} + #[cfg(test)] mod tests { use super::*; diff --git a/primitives/state-machine/Cargo.toml b/primitives/state-machine/Cargo.toml index 3752e95695..2408cce099 100644 --- a/primitives/state-machine/Cargo.toml +++ b/primitives/state-machine/Cargo.toml @@ -10,8 +10,8 @@ license = "GPL-3.0" log = "0.4.8" parking_lot = "0.10.0" hash-db = "0.15.2" -trie-db = "0.19.2" -trie-root = "0.15.2" +trie-db = "0.20.0" +trie-root = "0.16.0" sp-trie = { version = "2.0.0", path = "../trie" } sp-core = { version = "2.0.0", path = "../core" } sp-panic-handler = { version = "2.0.0", path = "../panic-handler" } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 4ef9b970ae..396ef6575a 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -213,6 +213,16 @@ pub trait Backend: std::fmt::Debug { fn usage_info(&self) -> UsageInfo { UsageInfo::empty() } + + /// Wipe the state database. + fn wipe(&self) -> Result<(), Self::Error> { + unimplemented!() + } + + /// Commit given transaction to storage. + fn commit(&self, _storage_root: H::Out, _transaction: Self::Transaction) -> Result<(), Self::Error> { + unimplemented!() + } } impl<'a, T: Backend, H: Hasher> Backend for &'a T { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 38a2e70262..53156cb186 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -583,6 +583,25 @@ where root.map(|r| r.map(|o| o.encode())) } + + fn wipe(&mut self) { + self.overlay.discard_prospective(); + self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache) + .expect(EXT_NOT_ALLOWED_TO_FAIL); + self.storage_transaction_cache.reset(); + self.backend.wipe().expect(EXT_NOT_ALLOWED_TO_FAIL) + } + + fn commit(&mut self) { + self.overlay.commit_prospective(); + let changes = self.overlay.drain_storage_changes(&self.backend, None, Default::default(), self.storage_transaction_cache) + .expect(EXT_NOT_ALLOWED_TO_FAIL); + self.backend.commit( + changes.transaction_storage_root, + changes.transaction, + ).expect(EXT_NOT_ALLOWED_TO_FAIL); + self.storage_transaction_cache.reset(); + } } impl<'a, H, B, N> sp_externalities::ExtensionStore for Ext<'a, H, N, B> diff --git a/primitives/state-machine/src/overlayed_changes.rs b/primitives/state-machine/src/overlayed_changes.rs index ed6f30a4f5..37187e163f 100644 --- a/primitives/state-machine/src/overlayed_changes.rs +++ b/primitives/state-machine/src/overlayed_changes.rs @@ -428,15 +428,18 @@ impl OverlayedChanges { /// /// Panics: /// Will panic if there are any uncommitted prospective changes. - pub fn into_committed(self) -> ( + fn drain_committed(&mut self) -> ( impl Iterator)>, impl Iterator)>, OwnedChildInfo))>, - ){ + ) { assert!(self.prospective.is_empty()); ( - self.committed.top.into_iter().map(|(k, v)| (k, v.value)), - self.committed.children.into_iter() - .map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))) + std::mem::replace(&mut self.committed.top, Default::default()) + .into_iter() + .map(|(k, v)| (k, v.value)), + std::mem::replace(&mut self.committed.children, Default::default()) + .into_iter() + .map(|(sk, (v, ci))| (sk, (v.into_iter().map(|(k, v)| (k, v.value)), ci))), ) } @@ -444,11 +447,22 @@ impl OverlayedChanges { pub fn into_storage_changes< B: Backend, H: Hasher, N: BlockNumber >( - self, + mut self, backend: &B, changes_trie_state: Option<&ChangesTrieState>, parent_hash: H::Out, mut cache: StorageTransactionCache, + ) -> Result, String> where H::Out: Ord + Encode + 'static { + self.drain_storage_changes(backend, changes_trie_state, parent_hash, &mut cache) + } + + /// Drain all changes into a [`StorageChanges`] instance. Leave empty overlay in place. + pub fn drain_storage_changes, H: Hasher, N: BlockNumber>( + &mut self, + backend: &B, + changes_trie_state: Option<&ChangesTrieState>, + parent_hash: H::Out, + mut cache: &mut StorageTransactionCache, ) -> Result, String> where H::Out: Ord + Encode + 'static { // If the transaction does not exist, we generate it. if cache.transaction.is_none() { @@ -474,7 +488,7 @@ impl OverlayedChanges { .take() .expect("Changes trie transaction was generated by `changes_trie_root`; qed"); - let (main_storage_changes, child_storage_changes) = self.into_committed(); + let (main_storage_changes, child_storage_changes) = self.drain_committed(); Ok(StorageChanges { main_storage_changes: main_storage_changes.collect(), diff --git a/primitives/state-machine/src/trie_backend_essence.rs b/primitives/state-machine/src/trie_backend_essence.rs index 2598682ae0..125a823f57 100644 --- a/primitives/state-machine/src/trie_backend_essence.rs +++ b/primitives/state-machine/src/trie_backend_essence.rs @@ -23,7 +23,7 @@ use log::{debug, warn}; use hash_db::{self, Hasher, EMPTY_PREFIX, Prefix}; use sp_trie::{Trie, MemoryDB, PrefixedMemoryDB, DBValue, default_child_trie_root, read_trie_value, read_child_trie_value, - for_keys_in_child_trie, KeySpacedDB}; + for_keys_in_child_trie, KeySpacedDB, TrieDBIterator}; use sp_trie::trie_types::{TrieDB, TrieError, Layout}; use crate::{backend::Consolidate, StorageKey, StorageValue}; use sp_core::storage::ChildInfo; @@ -252,16 +252,11 @@ impl, H: Hasher> TrieBackendEssence where H::Out: let mut iter = move |db| -> Result<(), Box>> { let trie = TrieDB::::new(db, root)?; - let mut iter = trie.iter()?; - iter.seek(prefix)?; - - for x in iter { + for x in TrieDBIterator::new_prefixed(&trie, prefix)? { let (key, value) = x?; - if !key.starts_with(prefix) { - break; - } + debug_assert!(key.starts_with(prefix)); f(&key, &value); } diff --git a/primitives/timestamp/Cargo.toml b/primitives/timestamp/Cargo.toml index fa146ebd6f..815aaf5305 100644 --- a/primitives/timestamp/Cargo.toml +++ b/primitives/timestamp/Cargo.toml @@ -12,6 +12,7 @@ sp-runtime = { version = "2.0.0", default-features = false, path = "../runtime" codec = { package = "parity-scale-codec", version = "1.0.0", default-features = false, features = ["derive"] } sp-inherents = { version = "2.0.0", default-features = false, path = "../inherents" } impl-trait-for-tuples = "0.1.3" +wasm-timer = "0.2" [features] default = [ "std" ] diff --git a/primitives/timestamp/src/lib.rs b/primitives/timestamp/src/lib.rs index 50b871d73e..979b98b495 100644 --- a/primitives/timestamp/src/lib.rs +++ b/primitives/timestamp/src/lib.rs @@ -90,7 +90,7 @@ impl ProvideInherentData for InherentDataProvider { &self, inherent_data: &mut InherentData, ) -> Result<(), sp_inherents::Error> { - use std::time::SystemTime; + use wasm_timer::SystemTime; let now = SystemTime::now(); now.duration_since(SystemTime::UNIX_EPOCH) diff --git a/primitives/transaction-pool/src/pool.rs b/primitives/transaction-pool/src/pool.rs index 0b23c27f82..8d8ff47cbe 100644 --- a/primitives/transaction-pool/src/pool.rs +++ b/primitives/transaction-pool/src/pool.rs @@ -29,7 +29,7 @@ use futures::{ use serde::{Deserialize, Serialize}; use sp_runtime::{ generic::BlockId, - traits::{Block as BlockT, Member, MaybeMallocSizeOf}, + traits::{Block as BlockT, Member}, transaction_validity::{ TransactionLongevity, TransactionPriority, TransactionTag, }, @@ -71,12 +71,18 @@ impl PoolStatus { /// - `Invalid` /// - `Usurped` /// - `Dropped` +/// 4. Re-entering the pool: +/// - `Retracted` +/// 5. Block finalized: +/// - `Finalized` +/// - `FinalityTimeout` /// /// The events will always be received in the order described above, however /// there might be cases where transactions alternate between `Future` and `Ready` /// pool, and are `Broadcast` in the meantime. /// /// There is also only single event causing the transaction to leave the pool. +/// I.e. only one of the listed ones should be triggered. /// /// Note that there are conditions that may cause transactions to reappear in the pool. /// 1. Due to possible forks, the transaction that ends up being in included @@ -86,8 +92,15 @@ impl PoolStatus { /// 3. `Invalid` transaction may become valid at some point in the future. /// (Note that runtimes are encouraged to use `UnknownValidity` to inform the pool about /// such case). +/// 4. `Retracted` transactions might be included in some next block. /// -/// However the user needs to re-subscribe to receive such notifications. +/// The stream is considered finished only when either `Finalized` or `FinalityTimeout` +/// event is triggered. You are however free to unsubscribe from notifications at any point. +/// The first one will be emitted when the block, in which transaction was included gets +/// finalized. The `FinalityTimeout` event will be emitted when the block did not reach finality +/// within 512 blocks. This either indicates that finality is not available for your chain, +/// or that finality gadget is lagging behind. If you choose to wait for finality longer, you can +/// re-subscribe for a particular transaction hash manually again. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub enum TransactionStatus { @@ -98,8 +111,14 @@ pub enum TransactionStatus { /// The transaction has been broadcast to the given peers. Broadcast(Vec), /// Transaction has been included in block with given hash. - #[serde(rename = "finalized")] // See #4438 InBlock(BlockHash), + /// The block this transaction was included in has been retracted. + Retracted(BlockHash), + /// Maximum number of finality watchers has been reached, + /// old watchers are being removed. + FinalityTimeout(BlockHash), + /// Transaction has been finalized by a finality-gadget, e.g GRANDPA + Finalized(BlockHash), /// Transaction has been replaced in the pool, by another transaction /// that provides the same tags. (e.g. same (sender, nonce)). Usurped(Hash), @@ -154,7 +173,7 @@ pub trait InPoolTransaction { } /// Transaction pool interface. -pub trait TransactionPool: Send + Sync + MaybeMallocSizeOf { +pub trait TransactionPool: Send + Sync { /// Block type. type Block: BlockT; /// Transaction hash type. @@ -217,11 +236,30 @@ pub trait TransactionPool: Send + Sync + MaybeMallocSizeOf { fn ready_transaction(&self, hash: &TxHash) -> Option>; } +/// Events that the transaction pool listens for. +pub enum ChainEvent { + /// New blocks have been added to the chain + NewBlock { + /// Is this the new best block. + is_new_best: bool, + /// Id of the just imported block. + id: BlockId, + /// Header of the just imported block + header: B::Header, + /// List of retracted blocks ordered by block number. + retracted: Vec, + }, + /// An existing block has been finalzied. + Finalized { + /// Hash of just finalized block + hash: B::Hash, + }, +} + /// Trait for transaction pool maintenance. -pub trait MaintainedTransactionPool : TransactionPool { +pub trait MaintainedTransactionPool: TransactionPool { /// Perform maintenance - fn maintain(&self, block: &BlockId, retracted: &[BlockHash]) - -> Pin + Send>>; + fn maintain(&self, event: ChainEvent) -> Pin + Send>>; } /// An abstraction for transaction pool. @@ -263,4 +301,4 @@ impl OffchainSubmitTransaction for TPool { e )) } -} \ No newline at end of file +} diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index c71d3fb84c..0cf268856b 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -37,7 +37,7 @@ pub use trie_stream::TrieStream; pub use node_codec::NodeCodec; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ - Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, + Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; diff --git a/prml/doughnut/src/impls.rs b/prml/doughnut/src/impls.rs index 16bbf693fb..15893731ac 100644 --- a/prml/doughnut/src/impls.rs +++ b/prml/doughnut/src/impls.rs @@ -19,7 +19,7 @@ use sp_core::{ ed25519::{self}, sr25519::{self}, }; -use sp_std::{self, convert::{TryFrom, TryInto}, prelude::*}; +use sp_std::{self, convert::{TryFrom}, prelude::*}; use sp_runtime::{Doughnut}; use sp_runtime::traits::{PlugDoughnutApi, DoughnutApi, DoughnutVerify, SignedExtension, ValidationError, Verify, VerifyError}; use sp_runtime::transaction_validity::{InvalidTransaction, TransactionValidityError, ValidTransaction}; @@ -142,8 +142,7 @@ where return Err(InvalidTransaction::Custom(code).into()) } // Convert chain reported timestamp from milliseconds into seconds as per doughnut timestamp spec. - // The conversion operation cannot fail hence `unwrap()` qed - let now = Runtime::TimestampProvider::now() / 1000_u32.try_into().unwrap(); + let now = Runtime::TimestampProvider::now() / 1000_u32.into(); // Check doughnut is valid for use by `who` at the current timestamp if let Err(err) = PlugDoughnutApi::validate(self, who, now) { let code = match err { diff --git a/test-utils/runtime/transaction-pool/Cargo.toml b/test-utils/runtime/transaction-pool/Cargo.toml index 615886d987..72e81f5f19 100644 --- a/test-utils/runtime/transaction-pool/Cargo.toml +++ b/test-utils/runtime/transaction-pool/Cargo.toml @@ -9,6 +9,7 @@ license = "GPL-3.0" substrate-test-runtime-client = { version = "2.0.0", path = "../client" } parking_lot = "0.10.0" codec = { package = "parity-scale-codec", version = "1.0.0" } +sp-blockchain = { version = "2.0.0", path = "../../../primitives/blockchain" } sp-runtime = { version = "2.0.0", path = "../../../primitives/runtime" } sp-transaction-pool = { version = "2.0.0", path = "../../../primitives/transaction-pool" } sc-transaction-graph = { version = "2.0.0", path = "../../../client/transaction-pool/graph" } diff --git a/test-utils/runtime/transaction-pool/src/lib.rs b/test-utils/runtime/transaction-pool/src/lib.rs index 58c801d8d6..aedc7dc4c3 100644 --- a/test-utils/runtime/transaction-pool/src/lib.rs +++ b/test-utils/runtime/transaction-pool/src/lib.rs @@ -50,7 +50,7 @@ impl std::error::Error for Error { } #[derive(Default)] -struct ChainState { +pub struct ChainState { pub block_by_number: HashMap>, pub block_by_hash: HashMap>, pub header_by_number: HashMap, @@ -96,16 +96,24 @@ impl TestApi { } /// Push block as a part of canonical chain under given number. - pub fn push_block(&self, block_number: BlockNumber, xts: Vec) { + pub fn push_block(&self, block_number: BlockNumber, xts: Vec) -> Header { let mut chain = self.chain.write(); - chain.block_by_number.insert(block_number, xts); - chain.header_by_number.insert(block_number, Header { + chain.block_by_number.insert(block_number, xts.clone()); + let header = Header { number: block_number, digest: Default::default(), extrinsics_root: Default::default(), - parent_hash: Default::default(), + parent_hash: block_number + .checked_sub(1) + .and_then(|num| { + chain.header_by_number.get(&num) + .cloned().map(|h| h.hash()) + }).unwrap_or_default(), state_root: Default::default(), - }); + }; + chain.block_by_hash.insert(header.hash(), xts); + chain.header_by_number.insert(block_number, header.clone()); + header } /// Push a block without a number. @@ -116,6 +124,20 @@ impl TestApi { chain.block_by_hash.insert(block_hash, xts); } + pub fn push_fork_block_with_parent(&self, parent: Hash, xts: Vec) -> Header { + let mut chain = self.chain.write(); + let blocknum = chain.block_by_number.keys().max().expect("block_by_number shouldn't be empty"); + let header = Header { + number: *blocknum, + digest: Default::default(), + extrinsics_root: Default::default(), + parent_hash: parent, + state_root: Default::default(), + }; + chain.block_by_hash.insert(header.hash(), xts); + header + } + fn hash_and_length_inner(ex: &Extrinsic) -> (Hash, usize) { let encoded = ex.encode(); (BlakeTwo256::hash(&encoded), encoded.len()) @@ -136,6 +158,11 @@ impl TestApi { self.validation_requests.read().clone() } + /// get a reference to the chain state + pub fn chain(&self) -> &RwLock { + &self.chain + } + /// Increment nonce in the inner state. pub fn increment_nonce(&self, account: AccountId) { let mut chain = self.chain.write(); @@ -197,7 +224,12 @@ impl sc_transaction_graph::ChainApi for TestApi { ) -> Result>, Error> { Ok(match at { generic::BlockId::Hash(x) => Some(x.clone()), - _ => Some(Default::default()), + generic::BlockId::Number(num) => { + self.chain.read() + .header_by_number.get(num) + .map(|h| h.hash()) + .or_else(|| Some(Default::default())) + }, }) } @@ -209,10 +241,9 @@ impl sc_transaction_graph::ChainApi for TestApi { } fn block_body(&self, id: &BlockId) -> Self::BodyFuture { - futures::future::ready(Ok(if let BlockId::Number(num) = id { - self.chain.read().block_by_number.get(num).cloned() - } else { - None + futures::future::ready(Ok(match id { + BlockId::Number(num) => self.chain.read().block_by_number.get(num).cloned(), + BlockId::Hash(hash) => self.chain.read().block_by_hash.get(hash).cloned(), })) } } diff --git a/utils/browser/Cargo.toml b/utils/browser/Cargo.toml index 3afa49fc25..e52f630cb5 100644 --- a/utils/browser/Cargo.toml +++ b/utils/browser/Cargo.toml @@ -17,12 +17,15 @@ js-sys = "0.3.34" wasm-bindgen = "0.2.57" wasm-bindgen-futures = "0.4.7" kvdb-web = "0.4" -service = { version = "0.8", package = "sc-service", path = "../../client/service", default-features = false } -network = { package = "sc-network", path = "../../client/network" } -chain-spec = { package = "sc-chain-spec", path = "../../client/chain-spec" } +sc-informant = { version = "0.8", path = "../../client/informant" } +sc-service = { version = "0.8", path = "../../client/service", default-features = false } +sc-network = { path = "../../client/network" } +sc-chain-spec = { path = "../../client/chain-spec" } # Imported just for the `no_cc` feature clear_on_drop = { version = "0.2.3", features = ["no_cc"] } # Imported just for the `wasm-bindgen` feature rand6 = { package = "rand", version = "0.6", features = ["wasm-bindgen"] } rand = { version = "0.7", features = ["wasm-bindgen"] } +futures-timer = { version = "3.0.1", features = ["wasm-bindgen"]} +chrono = { version = "0.4", features = ["wasmbind"] } diff --git a/utils/browser/src/lib.rs b/utils/browser/src/lib.rs index d7ffdca1aa..d82f982e14 100644 --- a/utils/browser/src/lib.rs +++ b/utils/browser/src/lib.rs @@ -17,7 +17,7 @@ use futures01::sync::mpsc as mpsc01; use log::{debug, info}; use std::sync::Arc; -use service::{ +use sc_service::{ AbstractService, RpcSession, Roles, Configuration, config::{DatabaseConfig, KeystoreConfig}, ChainSpec, RuntimeGenesis }; @@ -25,7 +25,7 @@ use wasm_bindgen::prelude::*; use futures::{prelude::*, channel::{oneshot, mpsc}, future::{poll_fn, ok}, compat::*}; use std::task::Poll; use std::pin::Pin; -use chain_spec::Extension; +use sc_chain_spec::Extension; pub use libp2p::wasm_ext::{ExtTransport, ffi::Transport}; pub use console_error_panic_hook::set_once as set_console_error_panic_hook; @@ -46,10 +46,14 @@ where let transport = ExtTransport::new(transport); let mut config = Configuration::default(); - config.network.transport = network::config::TransportConfig::Normal { + config.network.boot_nodes = chain_spec.boot_nodes().to_vec(); + config.telemetry_endpoints = chain_spec.telemetry_endpoints().clone(); + config.chain_spec = Some(chain_spec); + config.network.transport = sc_network::config::TransportConfig::Normal { wasm_external_transport: Some(transport.clone()), allow_private_ipv4: true, enable_mdns: false, + use_yamux_flow_control: true, }; config.task_executor = Some(Arc::new(move |fut| { wasm_bindgen_futures::spawn_local(fut) @@ -82,6 +86,11 @@ struct RpcMessage { /// Create a Client object that connects to a service. pub fn start_client(mut service: impl AbstractService) -> Client { + // Spawn informant + wasm_bindgen_futures::spawn_local( + sc_informant::build(&service, sc_informant::OutputFormat::Plain).map(drop) + ); + // We dispatch a background task responsible for processing the service. // // The main action performed by the code below consists in polling the service with diff --git a/utils/frame/rpc/system/src/lib.rs b/utils/frame/rpc/system/src/lib.rs index d26821caf2..9cc01fb6ec 100644 --- a/utils/frame/rpc/system/src/lib.rs +++ b/utils/frame/rpc/system/src/lib.rs @@ -87,7 +87,7 @@ where P: TransactionPool + 'static, Block: traits::Block, AccountId: Clone + std::fmt::Display + Codec, - Index: Clone + std::fmt::Display + Codec + Send + traits::SimpleArithmetic + 'static, + Index: Clone + std::fmt::Display + Codec + Send + traits::AtLeast32Bit + 'static, { fn nonce(&self, account: AccountId) -> FutureResult { let get_nonce = || { @@ -141,7 +141,7 @@ where F: Fetcher + 'static, Block: traits::Block, AccountId: Clone + std::fmt::Display + Codec + Send + 'static, - Index: Clone + std::fmt::Display + Codec + Send + traits::SimpleArithmetic + 'static, + Index: Clone + std::fmt::Display + Codec + Send + traits::AtLeast32Bit + 'static, { fn nonce(&self, account: AccountId) -> FutureResult { let best_hash = self.client.info().best_hash; @@ -189,7 +189,7 @@ fn adjust_nonce( ) -> Index where P: TransactionPool, AccountId: Clone + std::fmt::Display + Encode, - Index: Clone + std::fmt::Display + Encode + traits::SimpleArithmetic + 'static, + Index: Clone + std::fmt::Display + Encode + traits::AtLeast32Bit + 'static, { log::debug!(target: "rpc", "State nonce for {}: {}", account, nonce); // Now we need to query the transaction pool diff --git a/utils/grafana-data-source/Cargo.toml b/utils/grafana-data-source/Cargo.toml index ab4251ef57..e1bd037009 100644 --- a/utils/grafana-data-source/Cargo.toml +++ b/utils/grafana-data-source/Cargo.toml @@ -16,7 +16,7 @@ serde = { version = "1", features = ["derive"] } chrono = { version = "0.4", features = ["serde"] } lazy_static = "1.4" parking_lot = "0.10.0" -futures-timer = "2.0" +futures-timer = "3.0.1" derive_more = "0.99" [target.'cfg(not(target_os = "unknown"))'.dependencies] diff --git a/utils/grafana-data-source/test/Cargo.toml b/utils/grafana-data-source/test/Cargo.toml index 9575866fa8..18c080c8d1 100644 --- a/utils/grafana-data-source/test/Cargo.toml +++ b/utils/grafana-data-source/test/Cargo.toml @@ -9,5 +9,5 @@ edition = "2018" [dependencies] grafana-data-source = { version = "0.8", path = ".." } futures = "0.3" -futures-timer = "2.0" +futures-timer = "3.0.1" rand = "0.7"