diff --git a/Cargo.lock b/Cargo.lock index 2bed3b2bb5f4d..792a30e8e5ee0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,6 +1470,7 @@ dependencies = [ "sc-executor", "sc-service", "sp-runtime", + "sp-state-machine", "structopt", ] @@ -3426,6 +3427,7 @@ dependencies = [ "sc-executor", "sp-application-crypto", "sp-core", + "sp-externalities", "sp-io", "sp-runtime", "sp-state-machine", @@ -5650,6 +5652,8 @@ dependencies = [ "sp-core", "sp-runtime", "sp-state-machine", + "sp-trie", + "substrate-test-runtime-client", ] [[package]] @@ -6963,6 +6967,7 @@ dependencies = [ "sp-api", "sp-blockchain", "sp-consensus", + "sp-core", "sp-runtime", "sp-state-machine", "sp-version", diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index c8ef75a5a2a9a..a77efcf7f8526 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -36,6 +36,7 @@ pallet-transaction-payment = { version = "2.0.0-alpha.2", path = "../../../frame pallet-treasury = { version = "2.0.0-alpha.2", path = "../../../frame/treasury" } sp-application-crypto = { version = "2.0.0-alpha.2", path = "../../../primitives/application-crypto" } sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-externalities = { version = "0.8.0-alpha.3", path = "../../../primitives/externalities" } substrate-test-client = { version = "2.0.0-dev", path = "../../../test-utils/client" } wabt = "0.9.2" diff --git a/bin/node/executor/benches/bench.rs b/bin/node/executor/benches/bench.rs index ef0cdf445a561..cb6b6a0916841 100644 --- a/bin/node/executor/benches/bench.rs +++ b/bin/node/executor/benches/bench.rs @@ -25,8 +25,8 @@ use node_runtime::constants::currency::*; use node_testing::keyring::*; use sp_core::{NativeOrEncoded, NeverNativeValue}; use sp_core::storage::well_known_keys; -use sp_core::traits::CodeExecutor; -use frame_support::Hashable; +use sp_core::traits::{CodeExecutor, RuntimeCode}; +use frame_support::Hashable; use sp_state_machine::TestExternalities as CoreTestExternalities; use sc_executor::{NativeExecutor, RuntimeInfo, WasmExecutionMethod, Externalities}; use sp_runtime::traits::BlakeTwo256; @@ -90,9 +90,16 @@ fn construct_block( digest: Default::default(), }; + let runtime_code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(COMPACT_CODE.into()), + hash: vec![1, 2, 3], + heap_pages: None, + }; + // execute the block to get the real header. executor.call:: _>( ext, + &runtime_code, "Core_initialize_block", &header.encode(), true, @@ -102,6 +109,7 @@ fn construct_block( for i in extrinsics.iter() { executor.call:: _>( ext, + &runtime_code, "BlockBuilder_apply_extrinsic", &i.encode(), true, @@ -111,6 +119,7 @@ fn construct_block( let header = match executor.call:: _>( ext, + &runtime_code, "BlockBuilder_finalize_block", &[0u8;0], true, @@ -162,11 +171,16 @@ fn bench_execute_block(c: &mut Criterion) { ExecutionMethod::Wasm(wasm_method) => (false, *wasm_method), }; let executor = NativeExecutor::new(wasm_method, None); + let runtime_code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(COMPACT_CODE.into()), + hash: vec![1, 2, 3], + heap_pages: None, + }; // Get the runtime version to initialize the runtimes cache. { let mut test_ext = new_test_ext(&genesis_config); - executor.runtime_version(&mut test_ext.ext()).unwrap(); + executor.runtime_version(&mut test_ext.ext(), &runtime_code).unwrap(); } let blocks = test_blocks(&genesis_config, &executor); @@ -177,6 +191,7 @@ fn bench_execute_block(c: &mut Criterion) { for block in blocks.iter() { executor.call:: _>( &mut test_ext.ext(), + &runtime_code, "Core_execute_block", &block.0, use_native, diff --git a/bin/node/executor/tests/common.rs b/bin/node/executor/tests/common.rs index 34f3034208a08..d8ede1d7cda74 100644 --- a/bin/node/executor/tests/common.rs +++ b/bin/node/executor/tests/common.rs @@ -17,7 +17,7 @@ use codec::{Encode, Decode}; use frame_support::Hashable; use sp_state_machine::TestExternalities as CoreTestExternalities; -use sp_core::{NeverNativeValue, NativeOrEncoded, traits::CodeExecutor}; +use sp_core::{NeverNativeValue, NativeOrEncoded, traits::{CodeExecutor, RuntimeCode}}; use sp_runtime::{ApplyExtrinsicResult, traits::{Header as HeaderT, BlakeTwo256}}; use sc_executor::{NativeExecutor, WasmExecutionMethod}; use sc_executor::error::Result; @@ -29,6 +29,7 @@ use node_runtime::{ }; use node_primitives::{Hash, BlockNumber}; use node_testing::keyring::*; +use sp_externalities::Externalities; /// The wasm runtime code. /// @@ -71,8 +72,18 @@ pub fn executor_call< native_call: Option, ) -> (Result>, bool) { let mut t = t.ext(); + + let code = t.storage(sp_core::storage::well_known_keys::CODE).unwrap(); + let heap_pages = t.storage(sp_core::storage::well_known_keys::HEAP_PAGES); + let runtime_code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(code.as_slice().into()), + hash: sp_core::blake2_256(&code).to_vec(), + heap_pages: heap_pages.and_then(|hp| Decode::decode(&mut &hp[..]).ok()), + }; + executor().call::( &mut t, + &runtime_code, method, data, use_native, diff --git a/client/basic-authorship/src/basic_authorship.rs b/client/basic-authorship/src/basic_authorship.rs index 9dfe1c33be0fc..fc9a5dec0668d 100644 --- a/client/basic-authorship/src/basic_authorship.rs +++ b/client/basic-authorship/src/basic_authorship.rs @@ -471,4 +471,3 @@ mod tests { client.import(BlockOrigin::Own, block).unwrap(); } } - diff --git a/client/block-builder/Cargo.toml b/client/block-builder/Cargo.toml index 745669c033e38..dd4ebcb07f889 100644 --- a/client/block-builder/Cargo.toml +++ b/client/block-builder/Cargo.toml @@ -19,3 +19,7 @@ sp-core = { version = "2.0.0-alpha.2", path = "../../primitives/core" } sp-block-builder = { version = "2.0.0-alpha.2", path = "../../primitives/block-builder" } sc-client-api = { version = "2.0.0-alpha.2", path = "../api" } codec = { package = "parity-scale-codec", version = "1.2.0", features = ["derive"] } + +[dev-dependencies] +substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } +sp-trie = { version = "2.0.0-alpha.2", path = "../../primitives/trie" } diff --git a/client/block-builder/src/lib.rs b/client/block-builder/src/lib.rs index 6aaa8e9016270..d0e9f85ff4391 100644 --- a/client/block-builder/src/lib.rs +++ b/client/block-builder/src/lib.rs @@ -205,3 +205,41 @@ where }) } } + +#[cfg(test)] +mod tests { + use super::*; + use sp_blockchain::HeaderBackend; + use sp_core::Blake2Hasher; + use sp_state_machine::Backend; + use substrate_test_runtime_client::{DefaultTestClientBuilderExt, TestClientBuilderExt}; + + #[test] + fn block_building_storage_proof_does_not_include_runtime_by_default() { + let builder = substrate_test_runtime_client::TestClientBuilder::new(); + let backend = builder.backend(); + let client = builder.build(); + + let block = BlockBuilder::new( + &client, + client.info().best_hash, + client.info().best_number, + RecordProof::Yes, + Default::default(), + &*backend, + ).unwrap().build().unwrap(); + + let proof = block.proof.expect("Proof is build on request"); + + let backend = sp_state_machine::create_proof_check_backend::( + block.storage_changes.transaction_storage_root, + proof, + ).unwrap(); + + assert!( + backend.storage(&sp_core::storage::well_known_keys::CODE) + .unwrap_err() + .contains("Database missing expected key"), + ); + } +} diff --git a/client/executor/src/integration_tests/mod.rs b/client/executor/src/integration_tests/mod.rs index e787b229ec85c..28e18bdb00757 100644 --- a/client/executor/src/integration_tests/mod.rs +++ b/client/executor/src/integration_tests/mod.rs @@ -49,6 +49,7 @@ fn call_in_wasm( ); executor.call_in_wasm( &WASM_BINARY[..], + None, function, call_data, ext, @@ -513,6 +514,7 @@ fn should_trap_when_heap_exhausted(wasm_method: WasmExecutionMethod) { ); executor.call_in_wasm( &WASM_BINARY[..], + None, "test_exhaust_heap", &[0], &mut ext.ext(), diff --git a/client/executor/src/lib.rs b/client/executor/src/lib.rs index 1c21026fd8d35..539c8e82031e0 100644 --- a/client/executor/src/lib.rs +++ b/client/executor/src/lib.rs @@ -52,8 +52,12 @@ pub trait RuntimeInfo { /// Native runtime information. fn native_version(&self) -> &NativeVersion; - /// Extract RuntimeVersion of given :code block - fn runtime_version(&self, ext: &mut dyn Externalities) -> error::Result; + /// Extract [`RuntimeVersion`](sp_version::RuntimeVersion) of the given `runtime_code`. + fn runtime_version( + &self, + ext: &mut dyn Externalities, + runtime_code: &sp_core::traits::RuntimeCode, + ) -> error::Result; } #[cfg(test)] @@ -77,6 +81,7 @@ mod tests { ); let res = executor.call_in_wasm( &WASM_BINARY[..], + None, "test_empty_return", &[], &mut ext, diff --git a/client/executor/src/native_executor.rs b/client/executor/src/native_executor.rs index dfc88d2ede746..fb18528b60948 100644 --- a/client/executor/src/native_executor.rs +++ b/client/executor/src/native_executor.rs @@ -16,11 +16,11 @@ use crate::{ RuntimeInfo, error::{Error, Result}, - wasm_runtime::{RuntimeCache, WasmExecutionMethod, CodeSource}, + wasm_runtime::{RuntimeCache, WasmExecutionMethod}, }; use sp_version::{NativeVersion, RuntimeVersion}; use codec::{Decode, Encode}; -use sp_core::{NativeOrEncoded, traits::{CodeExecutor, Externalities}}; +use sp_core::{NativeOrEncoded, traits::{CodeExecutor, Externalities, RuntimeCode}}; use log::trace; use std::{result, panic::{UnwindSafe, AssertUnwindSafe}, sync::Arc}; use sp_wasm_interface::{HostFunctions, Function}; @@ -111,7 +111,7 @@ impl WasmExecutor { } } - /// Execute the given closure `f` with the latest runtime (based on the `CODE` key in `ext`). + /// Execute the given closure `f` with the latest runtime (based on `runtime_code`). /// /// The closure `f` is expected to return `Err(_)` when there happened a `panic!` in native code /// while executing the runtime in Wasm. If a `panic!` occurred, the runtime is invalidated to @@ -124,9 +124,9 @@ impl WasmExecutor { /// runtime is invalidated on any `panic!` to prevent a poisoned state. `ext` is already /// implicitly handled as unwind safe, as we store it in a global variable while executing the /// native runtime. - fn with_instance<'c, R, F>( + fn with_instance( &self, - code: CodeSource<'c>, + runtime_code: &RuntimeCode, ext: &mut dyn Externalities, f: F, ) -> Result @@ -137,7 +137,7 @@ impl WasmExecutor { ) -> Result>, { match self.cache.with_instance( - code, + runtime_code, ext, self.method, self.default_heap_pages, @@ -158,17 +158,48 @@ impl WasmExecutor { impl sp_core::traits::CallInWasm for WasmExecutor { fn call_in_wasm( &self, - wasm_blob: &[u8], + wasm_code: &[u8], + code_hash: Option>, method: &str, call_data: &[u8], ext: &mut dyn Externalities, ) -> std::result::Result, String> { - self.with_instance(CodeSource::Custom(wasm_blob), ext, |instance, _, mut ext| { + if let Some(hash) = code_hash { + let code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(wasm_code.into()), + hash, + heap_pages: None, + }; + + self.with_instance(&code, ext, |instance, _, mut ext| { + with_externalities_safe( + &mut **ext, + move || instance.call(method, call_data), + ) + }).map_err(|e| e.to_string()) + } else { + let module = crate::wasm_runtime::create_wasm_runtime_with_code( + self.method, + self.default_heap_pages, + &wasm_code, + self.host_functions.to_vec(), + self.allow_missing_func_imports, + ) + .map_err(|e| format!("Failed to create module: {:?}", e))?; + + let instance = module.new_instance() + .map_err(|e| format!("Failed to create instance: {:?}", e))?; + + let instance = AssertUnwindSafe(instance); + let mut ext = AssertUnwindSafe(ext); + with_externalities_safe( &mut **ext, move || instance.call(method, call_data), ) - }).map_err(|e| e.to_string()) + .and_then(|r| r) + .map_err(|e| e.to_string()) + } } } @@ -220,8 +251,11 @@ impl RuntimeInfo for NativeExecutor { fn runtime_version( &self, ext: &mut dyn Externalities, + runtime_code: &RuntimeCode, ) -> Result { - self.wasm.with_instance(CodeSource::Externalities, ext, + self.wasm.with_instance( + runtime_code, + ext, |_instance, version, _ext| Ok(version.cloned().ok_or_else(|| Error::ApiError("Unknown version".into()))) ) @@ -237,6 +271,7 @@ impl CodeExecutor for NativeExecutor { >( &self, ext: &mut dyn Externalities, + runtime_code: &RuntimeCode, method: &str, data: &[u8], use_native: bool, @@ -244,7 +279,7 @@ impl CodeExecutor for NativeExecutor { ) -> (Result>, bool) { let mut used_native = false; let result = self.wasm.with_instance( - CodeSource::Externalities, + runtime_code, ext, |instance, onchain_version, mut ext| { let onchain_version = onchain_version.ok_or_else( @@ -324,11 +359,12 @@ impl sp_core::traits::CallInWasm for NativeExecutor< fn call_in_wasm( &self, wasm_blob: &[u8], + code_hash: Option>, method: &str, call_data: &[u8], ext: &mut dyn Externalities, ) -> std::result::Result, String> { - sp_core::traits::CallInWasm::call_in_wasm(&self.wasm, wasm_blob, method, call_data, ext) + self.wasm.call_in_wasm(wasm_blob, code_hash, method, call_data, ext) } } diff --git a/client/executor/src/wasm_runtime.rs b/client/executor/src/wasm_runtime.rs index 680bc4c4589b0..09ed456e65959 100644 --- a/client/executor/src/wasm_runtime.rs +++ b/client/executor/src/wasm_runtime.rs @@ -20,11 +20,10 @@ //! components of the runtime that are expensive to initialize. use std::sync::Arc; -use std::borrow::Cow; use crate::error::{Error, WasmError}; use parking_lot::{Mutex, RwLock}; use codec::Decode; -use sp_core::{storage::well_known_keys, traits::Externalities}; +use sp_core::traits::{Externalities, RuntimeCode, FetchRuntimeCode}; use sp_version::RuntimeVersion; use std::panic::AssertUnwindSafe; use sc_executor_common::wasm_runtime::{WasmModule, WasmInstance}; @@ -41,14 +40,6 @@ pub enum WasmExecutionMethod { Compiled, } -/// Executoed code origin. -pub enum CodeSource<'a> { - /// Take code from storage, - Externalities, - /// Use provided code, - Custom(&'a [u8]), -} - /// A Wasm runtime object along with its cached runtime version. struct VersionedRuntime { /// Runtime code hash. @@ -102,8 +93,7 @@ impl RuntimeCache { /// /// `code` - Provides external code or tells the executor to fetch it from storage. /// - /// `ext` - Externalities to use for the runtime. This is used for setting - /// up an initial runtime instance. + /// `runtime_code` - The runtime wasm code used setup the runtime. /// /// `default_heap_pages` - Number of 64KB pages to allocate for Wasm execution. /// @@ -124,7 +114,7 @@ impl RuntimeCache { /// identifier `memory` can be found in the runtime. pub fn with_instance<'c, R, F>( &self, - code: CodeSource<'c>, + runtime_code: &'c RuntimeCode<'c>, ext: &mut dyn Externalities, wasm_method: WasmExecutionMethod, default_heap_pages: u64, @@ -138,28 +128,14 @@ impl RuntimeCache { &mut dyn Externalities) -> Result, { - let (code_hash, heap_pages) = match &code { - CodeSource::Externalities => { - ( - ext - .original_storage_hash(well_known_keys::CODE) - .ok_or(Error::InvalidCode("`CODE` not found in storage.".into()))?, - ext - .storage(well_known_keys::HEAP_PAGES) - .and_then(|pages| u64::decode(&mut &pages[..]).ok()) - .unwrap_or(default_heap_pages), - ) - }, - CodeSource::Custom(code) => { - (sp_core::blake2_256(code).to_vec(), default_heap_pages) - } - }; + let code_hash = &runtime_code.hash; + let heap_pages = runtime_code.heap_pages.unwrap_or(default_heap_pages); let mut runtimes = self.runtimes.lock(); // this must be released prior to calling f let pos = runtimes.iter().position(|r| r.as_ref().map_or( false, |r| r.wasm_method == wasm_method && - r.code_hash == code_hash && + r.code_hash == *code_hash && r.heap_pages == heap_pages )); @@ -168,19 +144,11 @@ impl RuntimeCache { .clone() .expect("`position` only returns `Some` for entries that are `Some`"), None => { - let code = match code { - CodeSource::Externalities => { - Cow::Owned(ext.original_storage(well_known_keys::CODE) - .ok_or(WasmError::CodeNotFound)?) - } - CodeSource::Custom(code) => { - Cow::Borrowed(code) - } - }; + let code = runtime_code.fetch_runtime_code().ok_or(WasmError::CodeNotFound)?; let result = create_versioned_wasm_runtime( &code, - code_hash, + code_hash.clone(), ext, wasm_method, heap_pages, diff --git a/client/network/src/chain.rs b/client/network/src/chain.rs index 3c075ec881ca1..12033e1d5125b 100644 --- a/client/network/src/chain.rs +++ b/client/network/src/chain.rs @@ -66,7 +66,12 @@ pub trait Client: Send + Sync { ) -> Result; /// Get method execution proof. - fn execution_proof(&self, block: &Block::Hash, method: &str, data: &[u8]) -> Result<(Vec, StorageProof), Error>; + fn execution_proof( + &self, + block: &Block::Hash, + method: &str, + data: &[u8], + ) -> Result<(Vec, StorageProof), Error>; /// Get key changes proof. fn key_changes_proof( diff --git a/client/network/src/protocol/light_client_handler.rs b/client/network/src/protocol/light_client_handler.rs index a141e134fca05..af8fa89f51015 100644 --- a/client/network/src/protocol/light_client_handler.rs +++ b/client/network/src/protocol/light_client_handler.rs @@ -254,13 +254,12 @@ where B: Block, { /// Construct a new light client handler. - pub fn new - ( cfg: Config - , chain: Arc> - , checker: Arc> - , peerset: sc_peerset::PeersetHandle - ) -> Self - { + pub fn new( + cfg: Config, + chain: Arc>, + checker: Arc>, + peerset: sc_peerset::PeersetHandle, + ) -> Self { LightClientHandler { config: cfg, chain, @@ -425,7 +424,8 @@ where log::trace!("remote call request from {} ({} at {:?})", peer, request.method, - request.block); + request.block, + ); let block = Decode::decode(&mut request.block.as_ref())?; @@ -436,7 +436,8 @@ where peer, request.method, request.block, - e); + e, + ); StorageProof::empty() } }; diff --git a/client/network/src/protocol/light_dispatch.rs b/client/network/src/protocol/light_dispatch.rs index b734fc91eb2d5..8146172e15908 100644 --- a/client/network/src/protocol/light_dispatch.rs +++ b/client/network/src/protocol/light_dispatch.rs @@ -750,7 +750,11 @@ pub mod tests { } } - fn check_execution_proof(&self, _: &RemoteCallRequest, _: StorageProof) -> ClientResult> { + fn check_execution_proof( + &self, + _: &RemoteCallRequest, + _: StorageProof, + ) -> ClientResult> { match self.ok { true => Ok(vec![42]), false => Err(ClientError::Backend("Test error".into())), diff --git a/client/src/call_executor.rs b/client/src/call_executor.rs index b5206d3c461a8..3fb6123640c5a 100644 --- a/client/src/call_executor.rs +++ b/client/src/call_executor.rs @@ -81,6 +81,7 @@ where id, self.backend.changes_trie_storage() )?; let state = self.backend.state_at(*id)?; + let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); let return_data = StateMachine::new( &state, changes_trie, @@ -89,6 +90,7 @@ where method, call_data, extensions.unwrap_or_default(), + &state_runtime_code.runtime_code()?, ).execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( strategy.get_manager(), None, @@ -135,42 +137,53 @@ where let mut storage_transaction_cache = storage_transaction_cache.map(|c| c.borrow_mut()); let mut state = self.backend.state_at(*at)?; + match recorder { - Some(recorder) => state.as_trie_backend() - .ok_or_else(|| - Box::new(sp_state_machine::ExecutionError::UnableToGenerateProof) - as Box + Some(recorder) => { + let trie_state = state.as_trie_backend() + .ok_or_else(|| + Box::new(sp_state_machine::ExecutionError::UnableToGenerateProof) as Box + )?; + + let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&trie_state); + // It is important to extract the runtime code here before we create the proof + // recorder. + let runtime_code = state_runtime_code.runtime_code()?; + + let backend = sp_state_machine::ProvingBackend::new_with_recorder( + trie_state, + recorder.clone(), + ); + + StateMachine::new( + &backend, + changes_trie_state, + &mut *changes.borrow_mut(), + &self.executor, + method, + call_data, + extensions.unwrap_or_default(), + &runtime_code, + ) + // TODO: https://github.com/paritytech/substrate/issues/4455 + // .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) + .execute_using_consensus_failure_handler(execution_manager, native_call) + }, + None => { + let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); + StateMachine::new( + &state, + changes_trie_state, + &mut *changes.borrow_mut(), + &self.executor, + method, + call_data, + extensions.unwrap_or_default(), + &state_runtime_code.runtime_code()?, ) - .and_then(|trie_state| { - let backend = sp_state_machine::ProvingBackend::new_with_recorder( - trie_state, - recorder.clone(), - ); - - StateMachine::new( - &backend, - changes_trie_state, - &mut *changes.borrow_mut(), - &self.executor, - method, - call_data, - extensions.unwrap_or_default(), - ) - // TODO: https://github.com/paritytech/substrate/issues/4455 - // .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) - .execute_using_consensus_failure_handler(execution_manager, native_call) - }), - None => StateMachine::new( - &state, - changes_trie_state, - &mut *changes.borrow_mut(), - &self.executor, - method, - call_data, - extensions.unwrap_or_default(), - ) - .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) - .execute_using_consensus_failure_handler(execution_manager, native_call) + .with_storage_transaction_cache(storage_transaction_cache.as_mut().map(|c| &mut **c)) + .execute_using_consensus_failure_handler(execution_manager, native_call) + } }.map_err(Into::into) } @@ -189,7 +202,8 @@ where changes_trie_state, None, ); - self.executor.runtime_version(&mut ext) + let state_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&state); + self.executor.runtime_version(&mut ext, &state_runtime_code.runtime_code()?) .map_err(|e| sp_blockchain::Error::VersionInvalid(format!("{:?}", e)).into()) } @@ -206,6 +220,7 @@ where &self.executor, method, call_data, + &sp_state_machine::backend::BackendRuntimeCode::new(trie_state).runtime_code()?, ) .map_err(Into::into) } diff --git a/client/src/client.rs b/client/src/client.rs index adfdfb4b631d2..ab5ea820eec16 100644 --- a/client/src/client.rs +++ b/client/src/client.rs @@ -41,8 +41,7 @@ use sp_runtime::{ use sp_state_machine::{ DBValue, Backend as StateBackend, ChangesTrieAnchorBlockId, prove_read, prove_child_read, ChangesTrieRootsStorage, ChangesTrieStorage, - ChangesTrieConfigurationRange, key_changes, key_changes_proof, StorageProof, - merge_storage_proofs, + ChangesTrieConfigurationRange, key_changes, key_changes_proof, }; use sc_executor::{RuntimeVersion, RuntimeInfo}; use sp_consensus::{ @@ -55,6 +54,7 @@ use sp_blockchain::{self as blockchain, well_known_cache_keys::Id as CacheKeyId, HeaderMetadata, CachedHeaderMetadata, }; +use sp_trie::StorageProof; use sp_api::{ CallApiAt, ConstructRuntimeApi, Core as CoreApi, ApiExt, ApiRef, ProvideRuntimeApi, @@ -472,7 +472,7 @@ impl Client where Ok(()) }, ())?; - Ok(merge_storage_proofs(proofs)) + Ok(StorageProof::merge(proofs)) } /// Generates CHT-based proof for roots of changes tries at given blocks (that are part of single CHT). @@ -1140,9 +1140,20 @@ impl ProofProvider for Client where method: &str, call_data: &[u8] ) -> sp_blockchain::Result<(Vec, StorageProof)> { + // Make sure we include the `:code` and `:heap_pages` in the execution proof to be + // backwards compatible. + // + // TODO: Remove when solved: https://github.com/paritytech/substrate/issues/5047 + let code_proof = self.read_proof( + id, + &mut [well_known_keys::CODE, well_known_keys::HEAP_PAGES].iter().map(|v| *v), + )?; + let state = self.state_at(id)?; let header = self.prepare_environment_block(id)?; - prove_execution(state, header, &self.executor, method, call_data) + prove_execution(state, header, &self.executor, method, call_data).map(|(r, p)| { + (r, StorageProof::merge(vec![p, code_proof])) + }) } fn header_proof(&self, id: &BlockId) -> sp_blockchain::Result<(Block::Header, StorageProof)> { diff --git a/client/src/genesis.rs b/client/src/genesis.rs index 0eecc6cdae890..6e55b5a42d397 100644 --- a/client/src/genesis.rs +++ b/client/src/genesis.rs @@ -89,6 +89,8 @@ mod tests { }; let hash = header.hash(); let mut overlay = OverlayedChanges::default(); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); StateMachine::new( backend, @@ -98,6 +100,7 @@ mod tests { "Core_initialize_block", &header.encode(), Default::default(), + &runtime_code, ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -111,6 +114,7 @@ mod tests { "BlockBuilder_apply_extrinsic", &tx.encode(), Default::default(), + &runtime_code, ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -124,6 +128,7 @@ mod tests { "BlockBuilder_finalize_block", &[], Default::default(), + &runtime_code, ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -161,6 +166,8 @@ mod tests { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); let _ = StateMachine::new( @@ -171,6 +178,7 @@ mod tests { "Core_execute_block", &b1data, Default::default(), + &runtime_code, ).execute( ExecutionStrategy::NativeElseWasm, ).unwrap(); @@ -189,6 +197,8 @@ mod tests { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); let _ = StateMachine::new( @@ -199,6 +209,7 @@ mod tests { "Core_execute_block", &b1data, Default::default(), + &runtime_code, ).execute( ExecutionStrategy::AlwaysWasm, ).unwrap(); @@ -217,6 +228,8 @@ mod tests { let backend = InMemoryBackend::from(storage); let (b1data, _b1hash) = block1(genesis_hash, &backend); + let backend_runtime_code = sp_state_machine::backend::BackendRuntimeCode::new(&backend); + let runtime_code = backend_runtime_code.runtime_code().expect("Code is part of the backend"); let mut overlay = OverlayedChanges::default(); let r = StateMachine::new( @@ -227,6 +240,7 @@ mod tests { "Core_execute_block", &b1data, Default::default(), + &runtime_code, ).execute( ExecutionStrategy::NativeElseWasm, ); diff --git a/client/src/light/backend.rs b/client/src/light/backend.rs index 6b5f9263009a2..749e24af046f7 100644 --- a/client/src/light/backend.rs +++ b/client/src/light/backend.rs @@ -172,9 +172,9 @@ impl ClientBackend for Backend> match maybe_val { Some(val) => self.blockchain.storage().insert_aux( &[(&key[..], &val[..])], - ::std::iter::empty(), + std::iter::empty(), )?, - None => self.blockchain.storage().insert_aux(::std::iter::empty(), &[&key[..]])?, + None => self.blockchain.storage().insert_aux(std::iter::empty(), &[&key[..]])?, } } } diff --git a/client/src/light/call_executor.rs b/client/src/light/call_executor.rs index cae5d5a0aa8e2..2e1a820f7c1ba 100644 --- a/client/src/light/call_executor.rs +++ b/client/src/light/call_executor.rs @@ -29,7 +29,6 @@ use sp_externalities::Extensions; use sp_state_machine::{ self, Backend as StateBackend, OverlayedChanges, ExecutionStrategy, create_proof_check_backend, execution_proof_check_on_trie_backend, ExecutionManager, StorageProof, - merge_storage_proofs, }; use hash_db::Hasher; @@ -206,7 +205,7 @@ pub fn prove_execution( method, call_data, )?; - let total_proof = merge_storage_proofs(vec![init_proof, exec_proof]); + let total_proof = StorageProof::merge(vec![init_proof, exec_proof]); Ok((result, total_proof)) } @@ -259,12 +258,18 @@ fn check_execution_proof_with_make_header( &trie_backend, &mut changes, executor, "Core_initialize_block", &next_header.encode(), + &runtime_code, )?; // execute method @@ -274,7 +279,9 @@ fn check_execution_proof_with_make_header> LightDataChecker { H::Out: Ord + codec::Codec, { // all the checks are sharing the same storage - let storage = create_proof_check_backend_storage(remote_roots_proof); + let storage = remote_roots_proof.into_memory_db(); // remote_roots.keys() are sorted => we can use this to group changes tries roots // that are belongs to the same CHT @@ -187,7 +187,8 @@ impl> LightDataChecker { local_cht_root, block, remote_changes_trie_root, - &proving_backend)?; + &proving_backend, + )?; // and return the storage to use in following checks storage = proving_backend.into_storage(); @@ -270,7 +271,7 @@ impl FetchChecker for LightDataChecker body: Vec ) -> ClientResult> { // TODO: #2621 - let extrinsics_root = HashFor::::ordered_trie_root( + let extrinsics_root = HashFor::::ordered_trie_root( body.iter().map(Encode::encode).collect(), ); if *request.header.extrinsics_root() == extrinsics_root { @@ -294,7 +295,7 @@ struct RootsStorage<'a, Number: AtLeast32Bit, Hash: 'a> { impl<'a, H, Number, Hash> ChangesTrieRootsStorage for RootsStorage<'a, Number, Hash> where H: Hasher, - Number: ::std::fmt::Display + ::std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, + Number: std::fmt::Display + std::hash::Hash + Clone + AtLeast32Bit + Encode + Decode + Send + Sync + 'static, Hash: 'a + Send + Sync + Clone + AsRef<[u8]>, { fn build_anchor( diff --git a/frame/system/benches/bench.rs b/frame/system/benches/bench.rs index cfcaa6f64ac59..90a4ad1d34de8 100644 --- a/frame/system/benches/bench.rs +++ b/frame/system/benches/bench.rs @@ -18,7 +18,7 @@ use criterion::{Criterion, criterion_group, criterion_main, black_box}; use frame_system as system; use frame_support::{decl_module, decl_event, impl_outer_origin, impl_outer_event, weights::Weight}; use sp_core::H256; -use sp_runtime::{Perbill, PerThing, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; +use sp_runtime::{Perbill, traits::{BlakeTwo256, IdentityLookup}, testing::Header}; mod module { use super::*; diff --git a/frame/system/src/lib.rs b/frame/system/src/lib.rs index f1286beac4e7b..c793933ce2024 100644 --- a/frame/system/src/lib.rs +++ b/frame/system/src/lib.rs @@ -1925,6 +1925,7 @@ mod tests { fn call_in_wasm( &self, _: &[u8], + _: Option>, _: &str, _: &[u8], _: &mut dyn sp_externalities::Externalities, diff --git a/primitives/api/test/Cargo.toml b/primitives/api/test/Cargo.toml index 6d2207c178a22..3b41e28cf3b2b 100644 --- a/primitives/api/test/Cargo.toml +++ b/primitives/api/test/Cargo.toml @@ -24,6 +24,7 @@ rustversion = "1.0.0" [dev-dependencies] criterion = "0.3.0" substrate-test-runtime-client = { version = "2.0.0-dev", path = "../../../test-utils/runtime/client" } +sp-core = { version = "2.0.0-alpha.1", path = "../../core" } [[bench]] name = "bench" diff --git a/primitives/api/test/tests/runtime_calls.rs b/primitives/api/test/tests/runtime_calls.rs index 18beaad9170af..7859202845e5c 100644 --- a/primitives/api/test/tests/runtime_calls.rs +++ b/primitives/api/test/tests/runtime_calls.rs @@ -164,6 +164,12 @@ fn record_proof_works() { let block_id = BlockId::Number(client.chain_info().best_number); let storage_root = longest_chain.best_chain().unwrap().state_root().clone(); + let runtime_code = sp_core::traits::RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(client.code_at(&block_id).unwrap().into()), + hash: vec![1], + heap_pages: None, + }; + let transaction = Transfer { amount: 1000, nonce: 0, @@ -192,5 +198,6 @@ fn record_proof_works() { &executor, "Core_execute_block", &block.encode(), + &runtime_code, ).expect("Executes block while using the proof backend"); } diff --git a/primitives/core/src/traits.rs b/primitives/core/src/traits.rs index e86a0234bf548..83cbebd7d6a92 100644 --- a/primitives/core/src/traits.rs +++ b/primitives/core/src/traits.rs @@ -19,9 +19,7 @@ use crate::{crypto::KeyTypeId, ed25519, sr25519}; use std::{ - fmt::{Debug, Display}, - panic::UnwindSafe, - sync::Arc, + fmt::{Debug, Display}, panic::UnwindSafe, sync::Arc, borrow::Cow, }; pub use sp_externalities::{Externalities, ExternalitiesExt}; @@ -97,6 +95,7 @@ pub trait CodeExecutor: Sized + Send + Sync + CallInWasm + Clone + 'static { >( &self, ext: &mut dyn Externalities, + runtime_code: &RuntimeCode, method: &str, data: &[u8], use_native: bool, @@ -104,15 +103,98 @@ pub trait CodeExecutor: Sized + Send + Sync + CallInWasm + Clone + 'static { ) -> (Result, Self::Error>, bool); } +/// Something that can fetch the runtime `:code`. +pub trait FetchRuntimeCode { + /// Fetch the runtime `:code`. + /// + /// If the `:code` could not be found/not available, `None` should be returned. + fn fetch_runtime_code<'a>(&'a self) -> Option>; +} + +/// Wrapper to use a `u8` slice or `Vec` as [`FetchRuntimeCode`]. +pub struct WrappedRuntimeCode<'a>(pub std::borrow::Cow<'a, [u8]>); + +impl<'a> FetchRuntimeCode for WrappedRuntimeCode<'a> { + fn fetch_runtime_code<'b>(&'b self) -> Option> { + Some(self.0.as_ref().into()) + } +} + +/// Type that implements [`FetchRuntimeCode`] and always returns `None`. +pub struct NoneFetchRuntimeCode; + +impl FetchRuntimeCode for NoneFetchRuntimeCode { + fn fetch_runtime_code<'a>(&'a self) -> Option> { + None + } +} + +/// The Wasm code of a Substrate runtime. +#[derive(Clone)] +pub struct RuntimeCode<'a> { + /// The code fetcher that can be used to lazily fetch the code. + pub code_fetcher: &'a dyn FetchRuntimeCode, + /// The optional heap pages this `code` should be executed with. + /// + /// If `None` are given, the default value of the executor will be used. + pub heap_pages: Option, + /// The SCALE encoded hash of `code`. + /// + /// The hashing algorithm isn't that important, as long as all runtime + /// code instances use the same. + pub hash: Vec, +} + +impl<'a> PartialEq for RuntimeCode<'a> { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl<'a> RuntimeCode<'a> { + /// Create an empty instance. + /// + /// This is only useful for tests that don't want to execute any code. + pub fn empty() -> Self { + Self { + code_fetcher: &NoneFetchRuntimeCode, + hash: Vec::new(), + heap_pages: None, + } + } +} + +impl<'a> FetchRuntimeCode for RuntimeCode<'a> { + fn fetch_runtime_code<'b>(&'b self) -> Option> { + self.code_fetcher.fetch_runtime_code() + } +} + +/// Could not find the `:code` in the externalities while initializing the [`RuntimeCode`]. +#[derive(Debug)] +pub struct CodeNotFound; + +impl std::fmt::Display for CodeNotFound { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + write!(f, "the storage entry `:code` doesn't have any code") + } +} + /// Something that can call a method in a WASM blob. pub trait CallInWasm: Send + Sync { /// Call the given `method` in the given `wasm_blob` using `call_data` (SCALE encoded arguments) /// to decode the arguments for the method. /// /// Returns the SCALE encoded return value of the method. + /// + /// # Note + /// + /// If `code_hash` is `Some(_)` the `wasm_code` module and instance will be cached internally, + /// otherwise it is thrown away after the call. fn call_in_wasm( &self, - wasm_blob: &[u8], + wasm_code: &[u8], + code_hash: Option>, method: &str, call_data: &[u8], ext: &mut dyn Externalities, diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index fa0f9e4454dd2..6fbd239b89cef 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -52,36 +52,6 @@ pub trait Externalities: ExtensionStore { key: &[u8], ) -> Option>; - /// Read original runtime storage, ignoring any overlayed changes. - fn original_storage(&self, key: &[u8]) -> Option>; - - /// Read original runtime child storage, ignoring any overlayed changes. - /// - /// Returns an `Option` that holds the SCALE encoded hash. - fn original_child_storage( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option>; - - /// Get original storage value hash, ignoring any overlayed changes. - /// This may be optimized for large values. - /// - /// Returns an `Option` that holds the SCALE encoded hash. - fn original_storage_hash(&self, key: &[u8]) -> Option>; - - /// Get original child storage value hash, ignoring any overlayed changes. - /// This may be optimized for large values. - /// - /// Returns an `Option` that holds the SCALE encoded hash. - fn original_child_storage_hash( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option>; - /// Read child runtime storage. /// /// Returns an `Option` that holds the SCALE encoded hash. diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 4b520a240a923..2702be02071af 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -366,7 +366,7 @@ pub trait Misc { self.extension::() .expect("No `CallInWasmExt` associated for the current context!") - .call_in_wasm(wasm, "Core_version", &[], &mut ext) + .call_in_wasm(wasm, None, "Core_version", &[], &mut ext) .ok() } } diff --git a/primitives/runtime-interface/test/src/lib.rs b/primitives/runtime-interface/test/src/lib.rs index 014a46e9d74ac..43eb9a80a685c 100644 --- a/primitives/runtime-interface/test/src/lib.rs +++ b/primitives/runtime-interface/test/src/lib.rs @@ -40,6 +40,7 @@ fn call_wasm_method(method: &str) -> TestExternalities { ); executor.call_in_wasm( &WASM_BINARY[..], + None, method, &[], &mut ext_ext, @@ -88,7 +89,7 @@ fn test_return_input_public_key() { #[test] #[should_panic( - expected = "\"Instantiation: Export ext_test_api_return_input_version_1 not found\"" + expected = "Instantiation: Export ext_test_api_return_input_version_1 not found" )] fn host_function_not_found() { call_wasm_method::<()>("test_return_data"); diff --git a/primitives/runtime/src/generic/block.rs b/primitives/runtime/src/generic/block.rs index a46396dce08f4..fb07d6c215d81 100644 --- a/primitives/runtime/src/generic/block.rs +++ b/primitives/runtime/src/generic/block.rs @@ -25,7 +25,10 @@ use serde::{Deserialize, Serialize}; use sp_std::prelude::*; use sp_core::RuntimeDebug; use crate::codec::{Codec, Encode, Decode}; -use crate::traits::{self, Member, Block as BlockT, Header as HeaderT, MaybeSerialize, MaybeMallocSizeOf}; +use crate::traits::{ + self, Member, Block as BlockT, Header as HeaderT, MaybeSerialize, MaybeMallocSizeOf, + NumberFor, +}; use crate::Justification; /// Something to identify a block. @@ -35,9 +38,9 @@ use crate::Justification; #[cfg_attr(feature = "std", serde(deny_unknown_fields))] pub enum BlockId { /// Identify by block header hash. - Hash(<::Header as HeaderT>::Hash), + Hash(Block::Hash), /// Identify by block number. - Number(<::Header as HeaderT>::Number), + Number(NumberFor), } impl BlockId { @@ -47,7 +50,7 @@ impl BlockId { } /// Create a block ID from a number. - pub fn number(number: ::Number) -> Self { + pub fn number(number: NumberFor) -> Self { BlockId::Number(number) } } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index ca6612a5e92dd..4fb59556e3fe4 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -18,9 +18,9 @@ use log::warn; use hash_db::Hasher; -use codec::Encode; +use codec::{Decode, Encode}; -use sp_core::storage::{ChildInfo, OwnedChildInfo}; +use sp_core::{traits::RuntimeCode, storage::{ChildInfo, OwnedChildInfo, well_known_keys}}; use sp_trie::{TrieMut, MemoryDB, trie_types::TrieDBMut}; use crate::{ @@ -359,3 +359,42 @@ pub(crate) fn insert_into_memory_db(mdb: &mut MemoryDB, input: I) -> Op Some(root) } + +/// Wrapper to create a [`RuntimeCode`] from a type that implements [`Backend`]. +pub struct BackendRuntimeCode<'a, B, H> { + backend: &'a B, + _marker: std::marker::PhantomData, +} + +impl<'a, B: Backend, H: Hasher> sp_core::traits::FetchRuntimeCode for + BackendRuntimeCode<'a, B, H> +{ + fn fetch_runtime_code<'b>(&'b self) -> Option> { + self.backend.storage(well_known_keys::CODE).ok().flatten().map(Into::into) + } +} + +impl<'a, B: Backend, H: Hasher> BackendRuntimeCode<'a, B, H> where H::Out: Encode { + /// Create a new instance. + pub fn new(backend: &'a B) -> Self { + Self { + backend, + _marker: std::marker::PhantomData, + } + } + + /// Return the [`RuntimeCode`] build from the wrapped `backend`. + pub fn runtime_code(&self) -> Result { + let hash = self.backend.storage_hash(well_known_keys::CODE) + .ok() + .flatten() + .ok_or("`:code` hash not found")? + .encode(); + let heap_pages = self.backend.storage(well_known_keys::HEAP_PAGES) + .ok() + .flatten() + .and_then(|d| Decode::decode(&mut &d[..]).ok()); + + Ok(RuntimeCode { code_fetcher: self, hash, heap_pages }) + } +} diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 7252ae10e9073..819244050ba4c 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -119,14 +119,6 @@ impl Externalities for BasicExternalities { self.storage(key).map(|v| Blake2Hasher::hash(&v).encode()) } - fn original_storage(&self, key: &[u8]) -> Option { - self.storage(key) - } - - fn original_storage_hash(&self, key: &[u8]) -> Option> { - self.storage_hash(key) - } - fn child_storage( &self, storage_key: ChildStorageKey, @@ -145,24 +137,6 @@ impl Externalities for BasicExternalities { self.child_storage(storage_key, child_info, key).map(|v| Blake2Hasher::hash(&v).encode()) } - fn original_child_storage_hash( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option> { - self.child_storage_hash(storage_key, child_info, key) - } - - fn original_child_storage( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option { - Externalities::child_storage(self, storage_key, child_info, key) - } - fn next_storage_key(&self, key: &[u8]) -> Option { let range = (Bound::Excluded(key), Bound::Unbounded); self.inner.top.range::<[u8], _>(range).next().map(|(k, _)| k).cloned() diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 53156cb18619c..bbb25355a8cce 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -179,30 +179,6 @@ where result.map(|r| r.encode()) } - fn original_storage(&self, key: &[u8]) -> Option { - let _guard = sp_panic_handler::AbortGuard::force_abort(); - let result = self.backend.storage(key).expect(EXT_NOT_ALLOWED_TO_FAIL); - - trace!(target: "state-trace", "{:04x}: GetOriginal {}={:?}", - self.id, - HexDisplay::from(&key), - result.as_ref().map(HexDisplay::from) - ); - result - } - - fn original_storage_hash(&self, key: &[u8]) -> Option> { - let _guard = sp_panic_handler::AbortGuard::force_abort(); - let result = self.backend.storage_hash(key).expect(EXT_NOT_ALLOWED_TO_FAIL); - - trace!(target: "state-trace", "{:04x}: GetOriginalHash {}={:?}", - self.id, - HexDisplay::from(&key), - result, - ); - result.map(|r| r.encode()) - } - fn child_storage( &self, storage_key: ChildStorageKey, @@ -253,47 +229,6 @@ where result.map(|r| r.encode()) } - fn original_child_storage( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option { - let _guard = sp_panic_handler::AbortGuard::force_abort(); - let result = self.backend - .child_storage(storage_key.as_ref(), child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL); - - trace!(target: "state-trace", "{:04x}: ChildOriginal({}) {}={:?}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&key), - result.as_ref().map(HexDisplay::from), - ); - - result - } - - fn original_child_storage_hash( - &self, - storage_key: ChildStorageKey, - child_info: ChildInfo, - key: &[u8], - ) -> Option> { - let _guard = sp_panic_handler::AbortGuard::force_abort(); - let result = self.backend - .child_storage_hash(storage_key.as_ref(), child_info, key) - .expect(EXT_NOT_ALLOWED_TO_FAIL); - - trace!(target: "state-trace", "{}: ChildHashOriginal({}) {}={:?}", - self.id, - HexDisplay::from(&storage_key.as_ref()), - HexDisplay::from(&key), - result, - ); - result.map(|r| r.encode()) - } - fn exists_storage(&self, key: &[u8]) -> bool { let _guard = sp_panic_handler::AbortGuard::force_abort(); let result = match self.overlay.storage(key) { @@ -820,21 +755,18 @@ mod tests { let ext = TestExt::new(&mut overlay, &mut cache, &backend, None, None); assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[10]), Some(vec![10])); - assert_eq!(ext.original_child_storage(child(), CHILD_INFO_1, &[10]), Some(vec![10])); assert_eq!( ext.child_storage_hash(child(), CHILD_INFO_1, &[10]), Some(Blake2Hasher::hash(&[10]).as_ref().to_vec()), ); assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[20]), None); - assert_eq!(ext.original_child_storage(child(), CHILD_INFO_1, &[20]), Some(vec![20])); assert_eq!( ext.child_storage_hash(child(), CHILD_INFO_1, &[20]), None, ); assert_eq!(ext.child_storage(child(), CHILD_INFO_1, &[30]), Some(vec![31])); - assert_eq!(ext.original_child_storage(child(), CHILD_INFO_1, &[30]), Some(vec![40])); assert_eq!( ext.child_storage_hash(child(), CHILD_INFO_1, &[30]), Some(Blake2Hasher::hash(&[31]).as_ref().to_vec()), diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index ff41237c83131..4d80ee37c98d1 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -23,8 +23,8 @@ use log::{warn, trace}; use hash_db::Hasher; use codec::{Decode, Encode, Codec}; use sp_core::{ - storage::ChildInfo, NativeOrEncoded, NeverNativeValue, - traits::{CodeExecutor, CallInWasmExt}, hexdisplay::HexDisplay, + storage::ChildInfo, NativeOrEncoded, NeverNativeValue, hexdisplay::HexDisplay, + traits::{CodeExecutor, CallInWasmExt, RuntimeCode}, }; use overlayed_changes::OverlayedChangeSet; use sp_externalities::Extensions; @@ -42,7 +42,7 @@ mod trie_backend; mod trie_backend_essence; mod stats; -pub use sp_trie::{trie_types::{Layout, TrieDBMut}, TrieMut, DBValue, MemoryDB}; +pub use sp_trie::{trie_types::{Layout, TrieDBMut}, StorageProof, TrieMut, DBValue, MemoryDB}; pub use testing::TestExternalities; pub use basic::BasicExternalities; pub use ext::Ext; @@ -67,8 +67,7 @@ pub use overlayed_changes::{ StorageCollection, ChildStorageCollection, }; pub use proving_backend::{ - create_proof_check_backend, create_proof_check_backend_storage, merge_storage_proofs, - ProofRecorder, ProvingBackend, ProvingBackendRecorder, StorageProof, + create_proof_check_backend, ProofRecorder, ProvingBackend, ProvingBackendRecorder, }; pub use trie_backend_essence::{TrieBackendStorage, Storage}; pub use trie_backend::TrieBackend; @@ -191,6 +190,7 @@ pub struct StateMachine<'a, B, H, N, Exec> changes_trie_state: Option>, _marker: PhantomData<(H, N)>, storage_transaction_cache: Option<&'a mut StorageTransactionCache>, + runtime_code: &'a RuntimeCode<'a>, } impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where @@ -209,6 +209,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where method: &'a str, call_data: &'a [u8], mut extensions: Extensions, + runtime_code: &'a RuntimeCode, ) -> Self { extensions.register(CallInWasmExt::new(exec.clone())); @@ -222,6 +223,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where changes_trie_state, _marker: PhantomData, storage_transaction_cache: None, + runtime_code, } } @@ -292,6 +294,7 @@ impl<'a, B, H, N, Exec> StateMachine<'a, B, H, N, Exec> where let (result, was_native) = self.exec.call( &mut ext, + self.runtime_code, self.method, self.call_data, use_native, @@ -436,6 +439,7 @@ pub fn prove_execution( exec: &Exec, method: &str, call_data: &[u8], + runtime_code: &RuntimeCode, ) -> Result<(Vec, StorageProof), Box> where B: Backend, @@ -446,7 +450,14 @@ where { let trie_backend = backend.as_trie_backend() .ok_or_else(|| Box::new(ExecutionError::UnableToGenerateProof) as Box)?; - prove_execution_on_trie_backend::<_, _, N, _>(trie_backend, overlay, exec, method, call_data) + prove_execution_on_trie_backend::<_, _, N, _>( + trie_backend, + overlay, + exec, + method, + call_data, + runtime_code, + ) } /// Prove execution using the given trie backend, overlayed changes, and call executor. @@ -464,6 +475,7 @@ pub fn prove_execution_on_trie_backend( exec: &Exec, method: &str, call_data: &[u8], + runtime_code: &RuntimeCode, ) -> Result<(Vec, StorageProof), Box> where S: trie_backend_essence::TrieBackendStorage, @@ -474,7 +486,14 @@ where { let proving_backend = proving_backend::ProvingBackend::new(trie_backend); let mut sm = StateMachine::<_, H, N, Exec>::new( - &proving_backend, None, overlay, exec, method, call_data, Extensions::default(), + &proving_backend, + None, + overlay, + exec, + method, + call_data, + Extensions::default(), + runtime_code, ); let result = sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( @@ -493,6 +512,7 @@ pub fn execution_proof_check( exec: &Exec, method: &str, call_data: &[u8], + runtime_code: &RuntimeCode, ) -> Result, Box> where H: Hasher, @@ -501,7 +521,14 @@ where N: crate::changes_trie::BlockNumber, { let trie_backend = create_proof_check_backend::(root.into(), proof)?; - execution_proof_check_on_trie_backend::<_, N, _>(&trie_backend, overlay, exec, method, call_data) + execution_proof_check_on_trie_backend::<_, N, _>( + &trie_backend, + overlay, + exec, + method, + call_data, + runtime_code, + ) } /// Check execution proof on proving backend, generated by `prove_execution` call. @@ -511,6 +538,7 @@ pub fn execution_proof_check_on_trie_backend( exec: &Exec, method: &str, call_data: &[u8], + runtime_code: &RuntimeCode, ) -> Result, Box> where H: Hasher, @@ -519,7 +547,14 @@ where N: crate::changes_trie::BlockNumber, { let mut sm = StateMachine::<_, H, N, Exec>::new( - trie_backend, None, overlay, exec, method, call_data, Extensions::default(), + trie_backend, + None, + overlay, + exec, + method, + call_data, + Extensions::default(), + runtime_code, ); sm.execute_using_consensus_failure_handler::<_, NeverNativeValue, fn() -> _>( @@ -692,7 +727,7 @@ mod tests { use super::*; use super::ext::Ext; use super::changes_trie::Configuration as ChangesTrieConfig; - use sp_core::{map, traits::Externalities, storage::ChildStorageKey}; + use sp_core::{map, traits::{Externalities, RuntimeCode}, storage::ChildStorageKey}; use sp_runtime::traits::BlakeTwo256; #[derive(Clone)] @@ -714,6 +749,7 @@ mod tests { >( &self, ext: &mut dyn Externalities, + _: &RuntimeCode, _method: &str, _data: &[u8], use_native: bool, @@ -755,6 +791,7 @@ mod tests { fn call_in_wasm( &self, _: &[u8], + _: Option>, _: &str, _: &[u8], _: &mut dyn Externalities, @@ -767,6 +804,7 @@ mod tests { fn execute_works() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, @@ -781,6 +819,7 @@ mod tests { "test", &[], Default::default(), + &wasm_code, ); assert_eq!( @@ -794,6 +833,7 @@ mod tests { fn execute_works_with_native_else_wasm() { let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, @@ -808,6 +848,7 @@ mod tests { "test", &[], Default::default(), + &wasm_code, ); assert_eq!(state_machine.execute(ExecutionStrategy::NativeElseWasm).unwrap(), vec![66]); @@ -818,6 +859,7 @@ mod tests { let mut consensus_failed = false; let backend = trie_backend::tests::test_trie(); let mut overlayed_changes = Default::default(); + let wasm_code = RuntimeCode::empty(); let mut state_machine = StateMachine::new( &backend, @@ -832,6 +874,7 @@ mod tests { "test", &[], Default::default(), + &wasm_code, ); assert!( @@ -864,6 +907,7 @@ mod tests { &executor, "test", &[], + &RuntimeCode::empty(), ).unwrap(); // check proof locally @@ -874,6 +918,7 @@ mod tests { &executor, "test", &[], + &RuntimeCode::empty(), ).unwrap(); // check that both results are correct diff --git a/primitives/state-machine/src/proving_backend.rs b/primitives/state-machine/src/proving_backend.rs index 7b6e8e0e69884..119fb59a7234e 100644 --- a/primitives/state-machine/src/proving_backend.rs +++ b/primitives/state-machine/src/proving_backend.rs @@ -18,19 +18,19 @@ use std::sync::Arc; use parking_lot::RwLock; -use codec::{Decode, Encode, Codec}; +use codec::{Decode, Codec}; use log::debug; use hash_db::{Hasher, HashDB, EMPTY_PREFIX, Prefix}; use sp_trie::{ MemoryDB, default_child_trie_root, read_trie_value_with, read_child_trie_value_with, - record_all_keys + record_all_keys, StorageProof, }; pub use sp_trie::Recorder; pub use sp_trie::trie_types::{Layout, TrieError}; use crate::trie_backend::TrieBackend; use crate::trie_backend_essence::{Ephemeral, TrieBackendEssence, TrieBackendStorage}; use crate::{Error, ExecutionError, Backend}; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use crate::DBValue; use sp_core::storage::ChildInfo; @@ -40,82 +40,6 @@ pub struct ProvingBackendRecorder<'a, S: 'a + TrieBackendStorage, H: 'a + Has pub(crate) proof_recorder: &'a mut Recorder, } -/// A proof that some set of key-value pairs are included in the storage trie. The proof contains -/// the storage values so that the partial storage backend can be reconstructed by a verifier that -/// does not already have access to the key-value pairs. -/// -/// The proof consists of the set of serialized nodes in the storage trie accessed when looking up -/// the keys covered by the proof. Verifying the proof requires constructing the partial trie from -/// the serialized nodes and performing the key lookups. -#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] -pub struct StorageProof { - trie_nodes: Vec>, -} - -impl StorageProof { - /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. - pub fn new(trie_nodes: Vec>) -> Self { - StorageProof { trie_nodes } - } - - /// Returns a new empty proof. - /// - /// An empty proof is capable of only proving trivial statements (ie. that an empty set of - /// key-value pairs exist in storage). - pub fn empty() -> Self { - StorageProof { - trie_nodes: Vec::new(), - } - } - - /// Returns whether this is an empty proof. - pub fn is_empty(&self) -> bool { - self.trie_nodes.is_empty() - } - - /// Create an iterator over trie nodes constructed from the proof. The nodes are not guaranteed - /// to be traversed in any particular order. - pub fn iter_nodes(self) -> StorageProofNodeIterator { - StorageProofNodeIterator::new(self) - } -} - -/// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to -/// be traversed in any particular order. -pub struct StorageProofNodeIterator { - inner: > as IntoIterator>::IntoIter, -} - -impl StorageProofNodeIterator { - fn new(proof: StorageProof) -> Self { - StorageProofNodeIterator { - inner: proof.trie_nodes.into_iter(), - } - } -} - -impl Iterator for StorageProofNodeIterator { - type Item = Vec; - - fn next(&mut self) -> Option { - self.inner.next() - } -} - -/// Merges multiple storage proofs covering potentially different sets of keys into one proof -/// covering all keys. The merged proof output may be smaller than the aggregate size of the input -/// proofs due to deduplication of trie nodes. -pub fn merge_storage_proofs(proofs: I) -> StorageProof - where I: IntoIterator -{ - let trie_nodes = proofs.into_iter() - .flat_map(|proof| proof.iter_nodes()) - .collect::>() - .into_iter() - .collect(); - StorageProof { trie_nodes } -} - impl<'a, S, H> ProvingBackendRecorder<'a, S, H> where S: TrieBackendStorage, @@ -222,7 +146,7 @@ impl<'a, S: 'a + TrieBackendStorage, H: 'a + Hasher> ProvingBackend<'a, S, H> let root = essence.root().clone(); let recorder = ProofRecorderBackend { backend: essence.backend_storage(), - proof_recorder: proof_recorder, + proof_recorder, }; ProvingBackend(TrieBackend::new(recorder, root)) } @@ -370,7 +294,7 @@ where H: Hasher, H::Out: Codec, { - let db = create_proof_check_backend_storage(proof); + let db = proof.into_memory_db(); if db.contains(&root, EMPTY_PREFIX) { Ok(TrieBackend::new(db, root)) @@ -379,20 +303,6 @@ where } } -/// Create in-memory storage of proof check backend. -pub fn create_proof_check_backend_storage( - proof: StorageProof, -) -> MemoryDB -where - H: Hasher, -{ - let mut db = MemoryDB::default(); - for item in proof.iter_nodes() { - db.insert(EMPTY_PREFIX, &item); - } - db -} - #[cfg(test)] mod tests { use crate::InMemoryBackend; diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index f6131c8ed5eed..80570a9792b10 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -21,6 +21,7 @@ mod error; mod node_header; mod node_codec; +mod storage_proof; mod trie_stream; use sp_std::boxed::Box; @@ -35,6 +36,7 @@ pub use error::Error; pub use trie_stream::TrieStream; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; +pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs new file mode 100644 index 0000000000000..254adc2fcb48a --- /dev/null +++ b/primitives/trie/src/storage_proof.rs @@ -0,0 +1,109 @@ +// Copyright 2020 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Parity is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Parity is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Parity. If not, see . + +use sp_std::vec::Vec; +use codec::{Encode, Decode}; +use hash_db::{Hasher, HashDB}; + +/// A proof that some set of key-value pairs are included in the storage trie. The proof contains +/// the storage values so that the partial storage backend can be reconstructed by a verifier that +/// does not already have access to the key-value pairs. +/// +/// The proof consists of the set of serialized nodes in the storage trie accessed when looking up +/// the keys covered by the proof. Verifying the proof requires constructing the partial trie from +/// the serialized nodes and performing the key lookups. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +pub struct StorageProof { + trie_nodes: Vec>, +} + +impl StorageProof { + /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. + pub fn new(trie_nodes: Vec>) -> Self { + StorageProof { trie_nodes } + } + + /// Returns a new empty proof. + /// + /// An empty proof is capable of only proving trivial statements (ie. that an empty set of + /// key-value pairs exist in storage). + pub fn empty() -> Self { + StorageProof { + trie_nodes: Vec::new(), + } + } + + /// Returns whether this is an empty proof. + pub fn is_empty(&self) -> bool { + self.trie_nodes.is_empty() + } + + /// Create an iterator over trie nodes constructed from the proof. The nodes are not guaranteed + /// to be traversed in any particular order. + pub fn iter_nodes(self) -> StorageProofNodeIterator { + StorageProofNodeIterator::new(self) + } + + /// Creates a `MemoryDB` from `Self`. + pub fn into_memory_db(self) -> crate::MemoryDB { + self.into() + } + + /// Merges multiple storage proofs covering potentially different sets of keys into one proof + /// covering all keys. The merged proof output may be smaller than the aggregate size of the input + /// proofs due to deduplication of trie nodes. + pub fn merge(proofs: I) -> Self where I: IntoIterator { + let trie_nodes = proofs.into_iter() + .flat_map(|proof| proof.iter_nodes()) + .collect::>() + .into_iter() + .collect(); + + Self { trie_nodes } + } +} + +/// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to +/// be traversed in any particular order. +pub struct StorageProofNodeIterator { + inner: > as IntoIterator>::IntoIter, +} + +impl StorageProofNodeIterator { + fn new(proof: StorageProof) -> Self { + StorageProofNodeIterator { + inner: proof.trie_nodes.into_iter(), + } + } +} + +impl Iterator for StorageProofNodeIterator { + type Item = Vec; + + fn next(&mut self) -> Option { + self.inner.next() + } +} + +impl From for crate::MemoryDB { + fn from(proof: StorageProof) -> Self { + let mut db = crate::MemoryDB::default(); + for item in proof.iter_nodes() { + db.insert(crate::EMPTY_PREFIX, &item); + } + db + } +} diff --git a/test-utils/runtime/src/system.rs b/test-utils/runtime/src/system.rs index 1e29f789dcb5f..296dd764800ca 100644 --- a/test-utils/runtime/src/system.rs +++ b/test-utils/runtime/src/system.rs @@ -342,7 +342,7 @@ mod tests { use sp_io::TestExternalities; use substrate_test_runtime_client::{AccountKeyring, Sr25519Keyring}; use crate::{Header, Transfer, WASM_BINARY}; - use sp_core::{NeverNativeValue, map, traits::CodeExecutor}; + use sp_core::{NeverNativeValue, map, traits::{CodeExecutor, RuntimeCode}}; use sc_executor::{NativeExecutor, WasmExecutionMethod, native_executor_instance}; use sp_io::hashing::twox_128; @@ -405,8 +405,15 @@ mod tests { fn block_import_works_wasm() { block_import_works(|b, ext| { let mut ext = ext.ext(); + let runtime_code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(WASM_BINARY.into()), + hash: Vec::new(), + heap_pages: None, + }; + executor().call:: _>( &mut ext, + &runtime_code, "Core_execute_block", &b.encode(), false, @@ -498,8 +505,15 @@ mod tests { fn block_import_with_transaction_works_wasm() { block_import_with_transaction_works(|b, ext| { let mut ext = ext.ext(); + let runtime_code = RuntimeCode { + code_fetcher: &sp_core::traits::WrappedRuntimeCode(WASM_BINARY.into()), + hash: Vec::new(), + heap_pages: None, + }; + executor().call:: _>( &mut ext, + &runtime_code, "Core_execute_block", &b.encode(), false, diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 93c62c3f9653e..89143ee9fe0ab 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -16,5 +16,6 @@ sc-client = { version = "0.8.0-alpha.2", path = "../../../client" } sc-client-db = { version = "0.8.0-alpha.2", path = "../../../client/db" } sc-executor = { version = "0.8.0-alpha.2", path = "../../../client/executor" } sp-runtime = { version = "2.0.0-alpha.2", path = "../../../primitives/runtime" } +sp-state-machine = { version = "0.8.0-alpha.2", path = "../../../primitives/state-machine" } structopt = "0.3.8" codec = { version = "1.2.0", package = "parity-scale-codec" } diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 899419e5de5ad..02c530abc129f 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -121,6 +121,7 @@ impl BenchmarkCmd { self.repeat, ).encode(), Default::default(), + &sp_state_machine::backend::BackendRuntimeCode::new(&state).runtime_code()?, ) .execute(strategy.into()) .map_err(|e| format!("Error executing runtime benchmark: {:?}", e))?;