diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000..ace14de91d142 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,88 @@ +stages: + - test + - build + +image: parity/rust:gitlab-ci + +variables: + CI_SERVER_NAME: "GitLab CI" + CARGO_HOME: "${CI_PROJECT_DIR}/cargo" + + BUILD_TARGET: ubuntu + BUILD_ARCH: amd64 + CARGO_TARGET: x86_64-unknown-linux-gnu + +cache: + key: "${CI_JOB_NAME}" + paths: + - ${CI_PROJECT_DIR}/target/ + - ${CI_PROJECT_DIR}/cargo/ + +.releaseable_branches: # list of git refs for building GitLab artifacts (think "pre-release binaries") + only: &releaseable_branches + - master + - stable + - beta + - tags + - gitlab-next + +.publishable_branches: # list of git refs for publishing builds to the "production" locations + only: &publishable_branches + - nightly # Our nightly builds from schedule, on `master` + - "v*" # Our version tags + - gitlab-next + +.collect_artifacts: &collect_artifacts + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}" + when: on_success + expire_in: 1 mos + paths: + - target/release/polkadot + +.determine_version: &determine_version | + export VERSION=$(grep -m 1 "version =" Cargo.toml | awk '{print $3}' | tr -d '"' | tr -d "\n") + echo "Version" $VERSION + + +#### stage: test + +test:rust:stable: &test + stage: test + variables: + RUST_TOOLCHAIN: stable + TARGET: native + script: + - ./scripts/init.sh + - export PATH="${CI_PROJECT_DIR}/cargo/bin/:$PATH" + - ./scripts/build.sh + - ./scripts/build-demos.sh + - time cargo test --all + tags: + - rust-stable + +.optional_test: &optional_test + <<: *test + allow_failure: true + only: + - master + +#### stage: build + +build:linux:ubuntu:amd64: &build + stage: build + only: *releaseable_branches + variables: + CARGO_TARGET: x86_64-unknown-linux-gnu + TARGET: native + RUST_TOOLCHAIN: stable + script: + - ./scripts/init.sh + - export PATH="${CI_PROJECT_DIR}/cargo/bin/:$PATH" + - ./scripts/build.sh + - ./scripts/build-demos.sh + - cargo build --release + <<: *collect_artifacts + tags: + - rust-stable + allow_failure: true diff --git a/Cargo.lock b/Cargo.lock index 17895bf28deb0..1b004feb8d0b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3197,7 +3197,7 @@ dependencies = [ "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", - "substrate-primitives 0.1.0", + "substrate-codec 0.1.0", "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/demo/executor/src/lib.rs b/demo/executor/src/lib.rs index 24ec4e5f573c9..6557f1e69ffeb 100644 --- a/demo/executor/src/lib.rs +++ b/demo/executor/src/lib.rs @@ -40,7 +40,7 @@ native_executor_instance!(pub Executor, demo_runtime::api::dispatch, demo_runtim #[cfg(test)] mod tests { - use runtime_io; + use runtime_io::{self, Externalities}; use super::Executor; use substrate_executor::{WasmExecutor, NativeExecutionDispatch}; use codec::{Encode, Decode, Joiner}; @@ -48,12 +48,12 @@ mod tests { use runtime_support::{Hashable, StorageValue, StorageMap}; use state_machine::{CodeExecutor, TestExternalities}; use primitives::twox_128; - use demo_primitives::{Hash, BlockNumber, AccountId}; + use demo_primitives::{Hash, BlockNumber, AccountId, Balance}; use runtime_primitives::traits::Header as HeaderT; use runtime_primitives::{ApplyOutcome, ApplyError, ApplyResult, MaybeUnsigned}; use {staking, system, consensus}; - use demo_runtime::{Header, Block, UncheckedExtrinsic, Extrinsic, Call, Concrete, Staking, - BuildStorage, GenesisConfig, SessionConfig, StakingConfig, BareExtrinsic}; + use demo_runtime::{Header, Block, UncheckedExtrinsic, Extrinsic, Call, Concrete, Staking, System, + BuildStorage, GenesisConfig, SessionConfig, StakingConfig, BareExtrinsic, SystemConfig}; use ed25519::{Public, Pair}; const BLOATY_CODE: &[u8] = include_bytes!("../../runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm"); @@ -182,12 +182,17 @@ mod tests { }); } - fn new_test_ext() -> TestExternalities { + fn new_test_ext(with_prefix: bool, existential_deposit: Balance) -> TestExternalities { use keyring::Keyring::*; let three = [3u8; 32].into(); GenesisConfig { consensus: Some(Default::default()), - system: Some(Default::default()), + system: Some(SystemConfig { + use_block_number_prefix: with_prefix, + storage_purge_interval: if with_prefix { Some(4) } else { None }, + min_purged_value_age: if with_prefix { Some(2) } else { None }, + ..Default::default() + }), session: Some(SessionConfig { session_length: 2, validators: vec![One.to_raw_public().into(), Two.to_raw_public().into(), three], @@ -202,7 +207,7 @@ mod tests { bonding_duration: 0, transaction_base_fee: 1, transaction_byte_fee: 0, - existential_deposit: 0, + existential_deposit, transfer_fee: 0, creation_fee: 0, reclaim_rebate: 0, @@ -212,7 +217,7 @@ mod tests { democracy: Some(Default::default()), council: Some(Default::default()), timestamp: Some(Default::default()), - }.build_storage().unwrap() + }.build_storage().unwrap().into() } fn construct_block(number: BlockNumber, parent_hash: Hash, state_root: Hash, extrinsics: Vec) -> (Vec, Hash) { @@ -243,11 +248,15 @@ mod tests { (Block { header, extrinsics }.encode(), hash.into()) } - fn block1() -> (Vec, Hash) { + fn block1(with_prefix: bool) -> (Vec, Hash) { construct_block( 1, [69u8; 32].into(), - hex!("b97d52254fc967bb94bed485de6a738e9fad05decfda3453711677b8becf6d0a").into(), + if with_prefix { + hex!("8c516ad4b624f1d2d83c4bff6a82681edff7ef226d547da0119b70301a172da4") + } else { + hex!("b97d52254fc967bb94bed485de6a738e9fad05decfda3453711677b8becf6d0a") + }.into(), vec![BareExtrinsic { signed: alice(), index: 0, @@ -259,7 +268,7 @@ mod tests { fn block2() -> (Vec, Hash) { construct_block( 2, - block1().1, + block1(false).1, hex!("a1f018d2faa339f72f5ee29050b4670d971e2e271cc06c41ee9cbe1f4c6feec9").into(), vec![ BareExtrinsic { @@ -291,9 +300,9 @@ mod tests { #[test] fn full_native_block_import_works() { - let mut t = new_test_ext(); + let mut t = new_test_ext(false, 0); - executor().call(&mut t, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap(); + executor().call(&mut t, COMPACT_CODE, "execute_block", &block1(false).0, true).0.unwrap(); runtime_io::with_externalities(&mut t, || { assert_eq!(Staking::voting_balance(&alice()), 41); @@ -310,9 +319,9 @@ mod tests { #[test] fn full_wasm_block_import_works() { - let mut t = new_test_ext(); + let mut t = new_test_ext(false, 0); - WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1().0, true).0.unwrap(); + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1(false).0, true).0.unwrap(); runtime_io::with_externalities(&mut t, || { assert_eq!(Staking::voting_balance(&alice()), 41); @@ -329,7 +338,7 @@ mod tests { #[test] fn wasm_big_block_import_fails() { - let mut t = new_test_ext(); + let mut t = new_test_ext(false, 0); let r = WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, true).0; assert!(!r.is_ok()); @@ -337,7 +346,7 @@ mod tests { #[test] fn native_big_block_import_succeeds() { - let mut t = new_test_ext(); + let mut t = new_test_ext(false, 0); let r = Executor::with_heap_pages(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, true).0; assert!(r.is_ok()); @@ -345,7 +354,7 @@ mod tests { #[test] fn native_big_block_import_fails_on_fallback() { - let mut t = new_test_ext(); + let mut t = new_test_ext(false, 0); let r = Executor::with_heap_pages(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1big().0, false).0; assert!(!r.is_ok()); @@ -395,4 +404,343 @@ mod tests { assert_eq!(Staking::voting_balance(&bob()), 69); }); } + + #[test] + fn change_block_number_is_unknown_by_runtime_without_prefix() { + let mut t = new_test_ext(false, 0); + runtime_io::with_externalities(&mut t, || { + assert_eq!(System::storage_value_change_block( + &>::key() + ), None) + }); + } + + #[test] + fn change_block_number_is_known_by_native_runtime_with_prefix() { + let mut t = new_test_ext(true, 0); + + runtime_io::with_externalities(&mut t, || { + // session reward is non-empty at genesis + assert_eq!(System::storage_value_change_block( + &>::key() + ), Some(0)); + // Alice have a non-empty balance at the genesis + assert_eq!(System::storage_value_change_block( + &>::key_for(alice()) + ), Some(0)); + // and Bob does not + assert_eq!(System::storage_value_change_block( + &>::key_for(bob()) + ), None); + }); + + executor().call(&mut t, COMPACT_CODE, "execute_block", &block1(true).0, true).0.unwrap(); + + runtime_io::with_externalities(&mut t, || { + // session reward has not changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key() + ), Some(0)); + // Alice balance has been changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key_for(alice()) + ), Some(1)); + // Bob balance has been changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key_for(bob()) + ), Some(1)); + }); + } + + #[test] + fn change_block_number_is_known_by_wasm_runtime_with_prefix() { + let mut t = new_test_ext(true, 0); + + runtime_io::with_externalities(&mut t, || { + // session reward is non-empty at genesis + assert_eq!(System::storage_value_change_block( + &>::key() + ), Some(0)); + // Alice have a non-empty balance at the genesis + assert_eq!(System::storage_value_change_block( + &>::key_for(alice()) + ), Some(0)); + // and Bob does not + assert_eq!(System::storage_value_change_block( + &>::key_for(bob()) + ), None); + }); + + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1(true).0, true).0.unwrap(); + + runtime_io::with_externalities(&mut t, || { + // session reward has not changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key() + ), Some(0)); + // Alice balance has been changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key_for(alice()) + ), Some(1)); + // Bob balance has been changed at block#1 + assert_eq!(System::storage_value_change_block( + &>::key_for(bob()) + ), Some(1)); + }); + } + + fn alice_balance_with_prefix(t: &TestExternalities) -> (Option>, Option>) { + t.storage(&twox_128(&>::key_for(alice()))) + .map(|mut prefix| { + let balance = prefix.split_off(8); + (Some(prefix), if balance.is_empty() { None } else { Some(balance) }) + }) + .unwrap_or_default() + } + + fn is_alice_balance_scheduled_for_purge(t: &TestExternalities) -> bool { + let mut key = b":deleted:map:".to_vec(); + key.extend(&twox_128(&>::key_for(alice()))); + t.storage(&key).is_some() + } + + fn purge_check_blocks() -> (Vec, Vec, Vec, Vec) { + let (block1, block1_hash) = construct_block(1, [0u8; 32].into(), + hex!("cf08615c8d3cd648d9342375ee68a4ba0e0c8c1a00f7588411f981acd54bb664").into(), + vec![BareExtrinsic { + signed: alice(), + index: 0, + function: Call::Staking(staking::Call::transfer(bob().into(), 100)), + }]); + let (block2, block2_hash) = construct_block(2, block1_hash, + hex!("7d81c7902acce2ab17043e657e5db5e86c7b2bf96ad6efebd8ab915d4824c0d5").into(), + vec![]); + let (block3, block3_hash) = construct_block(3, block2_hash, + hex!("2ceb56cc19833a285fce9c4e0a9b3c90a97483586bf2b3f8dbcae028e883a3d5").into(), + vec![]); + let (block4, _) = construct_block(4, block3_hash, + hex!("373319ec698d97e3a65b40e11dde0d89b64f8abdc084c881ebcdc23303125ee0").into(), + vec![]); + (block1, block2, block3, block4) + } + + #[test] + fn prefixed_values_are_purged_after_native_purge_block_import() { + let mut t = new_test_ext(true, 100); + let (block1, block2, block3, block4) = purge_check_blocks(); + + // at the genesis block there's Alice with free balance of 111 + // let's say at block1 Alice transfers 100 dots to Bob + // => Alice account disappers, because her balance is dropped below existential deposit (100) + executor().call(&mut t, COMPACT_CODE, "execute_block", &block1, true).0.unwrap(); + + // after block1 Alice' free balance is deleted, but it still has a footprint in db + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#2 + executor().call(&mut t, COMPACT_CODE, "execute_block", &block2, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#3 + executor().call(&mut t, COMPACT_CODE, "execute_block", &block3, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // purge is performed @ block#4 => mine empty block && check that footprint is removed + executor().call(&mut t, COMPACT_CODE, "execute_block", &block4, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (None, None)); + } + + #[test] + fn prefixed_values_are_purged_after_wasm_purge_block_import() { + let mut t = new_test_ext(true, 100); + let (block1, block2, block3, block4) = purge_check_blocks(); + + // at the genesis block there's Alice with free balance of 111 + // let's say at block1 Alice transfers 100 dots to Bob + // => Alice account disappers, because her balance is dropped below existential deposit (100) + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1, true).0.unwrap(); + + // after block1 Alice' free balance is deleted, but it still has a footprint in db + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#2 + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block2, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#3 + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block3, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // purge is performed @ block#4 => mine empty block && check that footprint is removed + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block4, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (None, None)); + } + + fn recently_purge_check_blocks() -> (Vec>, Vec>, Vec) { + let (block1, block1_hash) = construct_block(1, [0u8; 32].into(), + hex!("f08294d628fd15e6410184f0a87c867fdea6d13fc1da718696e753f96d93ae72").into(), + vec![]); + let (block2, block2_hash) = construct_block(2, block1_hash, + hex!("fd175eb2a1df58e4e103c1f630a11bdc46a4ea0bb26fcb44315a37c8694ce09a").into(), + vec![]); + let (block3, block3_hash) = construct_block(3, block2_hash, + hex!("98b2153929fca311a1edce644d1bdfd223405005e9c94f363441b2c8c617af48").into(), + vec![BareExtrinsic { + signed: alice(), + index: 0, + function: Call::Staking(staking::Call::transfer(bob().into(), 100)), + }]); + let (block4, block4_hash) = construct_block(4, block3_hash, + hex!("9f9c8a7ccef3e9b55f908a5797bba1515a7e13b0c1924c14e5379fdd60931cf3").into(), + vec![]); + let (block5, block5_hash) = construct_block(5, block4_hash, + hex!("7202b88a03c02581706634571171027b632a2e4285b9f70bfa2acaeada441b36").into(), + vec![]); + let (block6, block6_hash) = construct_block(6, block5_hash, + hex!("b345fb946ef7c8274647fc465ad4f27d06936ecf163bc7e966548a644ab05179").into(), + vec![]); + let (block7, block7_hash) = construct_block(7, block6_hash, + hex!("b6007eb860ba775a8039ffd427f42938f9d000f54e7ef119daa8add8e4c85ff5").into(), + vec![]); + let (block8, _) = construct_block(8, block7_hash, + hex!("c17bfe3a2bfe99a600c9f4e3582b04b46d7b2127797abf543be295eccb378795").into(), + vec![]); + (vec![block1, block2], vec![block3, block4, block5, block6, block7], block8) + } + + #[test] + fn prefixed_values_are_not_purged_after_native_purge_block_import_if_deleted_recently() { + let mut t = new_test_ext(true, 100); + let (has_balance_blocks, has_no_balance_blocks, has_no_footprint_block) = recently_purge_check_blocks(); + + // at these blocks Alice has a balance + for has_balance_block in has_balance_blocks { + executor().call(&mut t, COMPACT_CODE, "execute_block", &has_balance_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_some() && balance.is_some()); + } + + // at these blocks Alice has no balance, but balance is not yet purged + for has_no_balance_block in has_no_balance_blocks { + executor().call(&mut t, COMPACT_CODE, "execute_block", &has_no_balance_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_some() && balance.is_none()); + } + + // finally balance is purged off the db + executor().call(&mut t, COMPACT_CODE, "execute_block", &has_no_footprint_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_none() && balance.is_none()); + } + + #[test] + fn prefixed_values_are_not_purged_after_wasm_purge_block_import_if_deleted_recently() { + let mut t = new_test_ext(true, 100); + let (has_balance_blocks, has_no_balance_blocks, has_no_footprint_block) = recently_purge_check_blocks(); + + // at these blocks Alice has a balance + for has_balance_block in has_balance_blocks { + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &has_balance_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_some() && balance.is_some()); + } + + // at these blocks Alice has no balance, but balance is not yet purged + for has_no_balance_block in has_no_balance_blocks { + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &has_no_balance_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_some() && balance.is_none()); + } + + // finally balance is purged off the db + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &has_no_footprint_block, true).0.unwrap(); + let (prefix, balance) = alice_balance_with_prefix(&t); + assert!(prefix.is_none() && balance.is_none()); + } + + fn recreated_purge_check_blocks() -> (Vec, Vec, Vec, Vec) { + let (block1, block1_hash) = construct_block(1, [0u8; 32].into(), + hex!("68204a44f3f5da24c83ef51c09a7e121d7df744105d94bbe37f9429176f71ade").into(), + vec![BareExtrinsic { + signed: alice(), + index: 0, + function: Call::Staking(staking::Call::transfer(bob().into(), 110)), + }]); + let (block2, block2_hash) = construct_block(2, block1_hash, + hex!("024ce4045f676fc2b40eb6ba34a23552a06a4d7698f581ef53e9d9a09bb05eaf").into(), + vec![]); + let (block3, block3_hash) = construct_block(3, block2_hash, + hex!("2a990bbc72f6856422b68bcb1c096638529df5699ee74c94accbefd4d3fac5a2").into(), + vec![BareExtrinsic { + signed: bob(), + index: 0, + function: Call::Staking(staking::Call::transfer(alice().into(), 109)), + }]); + let (block4, _) = construct_block(4, block3_hash, + hex!("1667712e3060c5c246998326bfbf059f1f4a3beafdcf3e08e8cffb3583a057f8").into(), + vec![]); + (block1, block2, block3, block4) + } + + #[test] + fn prefixed_values_are_not_purged_after_native_purge_block_import_if_recreated_after_deletion() { + let mut t = new_test_ext(true, 100); + let (block1, block2, block3, block4) = recreated_purge_check_blocks(); + + // at the genesis block there's Alice with free balance of 111 + // let's say at block1 Alice transfers 100 dots to Bob + // => Alice account disappers, because her balance is dropped below existential deposit (100) + executor().call(&mut t, COMPACT_CODE, "execute_block", &block1, true).0.unwrap(); + + // after block1 Alice' free balance is deleted, but it still has a footprint in db + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#2 + executor().call(&mut t, COMPACT_CODE, "execute_block", &block2, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // at block#3 Bob returns 100 DOTs to Alice => Alice account is recreated back + executor().call(&mut t, COMPACT_CODE, "execute_block", &block3, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), + (Some(vec![3, 0, 0, 0, 0, 0, 0, 0]), Some(vec![109, 0, 0, 0, 0, 0, 0, 0]))); + + // purge is performed @ block#4 => Aice account is not deleted + executor().call(&mut t, COMPACT_CODE, "execute_block", &block4, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), + (Some(vec![3, 0, 0, 0, 0, 0, 0, 0]), Some(vec![109, 0, 0, 0, 0, 0, 0, 0]))); + + // check that value is not scheduled for purge anymore + assert!(!is_alice_balance_scheduled_for_purge(&t)); + } + + #[test] + fn prefixed_values_are_not_purged_after_wasm_purge_block_import_if_recreated_after_deletion() { + let mut t = new_test_ext(true, 100); + let (block1, block2, block3, block4) = recreated_purge_check_blocks(); + + // at the genesis block there's Alice with free balance of 111 + // let's say at block1 Alice transfers 110 dots to Bob + // => Alice account disappers, because her balance is dropped below existential deposit (100) + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block1, true).0.unwrap(); + + // after block1 Alice' free balance is deleted, but it still has a footprint in db + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // mine empty block#2 + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block2, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), (Some(vec![1, 0, 0, 0, 0, 0, 0, 0]), None)); + + // at block#3 Bob returns 109 DOTs to Alice => Alice account is recreated back + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block3, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), + (Some(vec![3, 0, 0, 0, 0, 0, 0, 0]), Some(vec![109, 0, 0, 0, 0, 0, 0, 0]))); + + // purge is performed @ block#4 => Aice account is not deleted + WasmExecutor::new(8, 8).call(&mut t, COMPACT_CODE, "execute_block", &block4, true).0.unwrap(); + assert_eq!(alice_balance_with_prefix(&t), + (Some(vec![3, 0, 0, 0, 0, 0, 0, 0]), Some(vec![109, 0, 0, 0, 0, 0, 0, 0]))); + + // check that value is not scheduled for purge anymore + assert!(!is_alice_balance_scheduled_for_purge(&t)); + } } diff --git a/demo/runtime/wasm/Cargo.lock b/demo/runtime/wasm/Cargo.lock index d5eac31f66037..b205c99d173b8 100644 --- a/demo/runtime/wasm/Cargo.lock +++ b/demo/runtime/wasm/Cargo.lock @@ -923,7 +923,6 @@ dependencies = [ "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-codec 0.1.0", - "substrate-primitives 0.1.0", "substrate-runtime-io 0.1.0", "substrate-runtime-primitives 0.1.0", "substrate-runtime-std 0.1.0", @@ -970,7 +969,7 @@ dependencies = [ "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", - "substrate-primitives 0.1.0", + "substrate-codec 0.1.0", "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm b/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm index 353df1156ad7d..2270cc32272f1 100644 Binary files a/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm and b/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.compact.wasm differ diff --git a/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm b/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm index 604500c98b590..886878a379821 100755 Binary files a/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm and b/demo/runtime/wasm/target/wasm32-unknown-unknown/release/demo_runtime.wasm differ diff --git a/polkadot/runtime/src/parachains.rs b/polkadot/runtime/src/parachains.rs index 4ac5f3f2a1089..ba3047e244ade 100644 --- a/polkadot/runtime/src/parachains.rs +++ b/polkadot/runtime/src/parachains.rs @@ -214,7 +214,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl runtime_primitives::BuildStorage for GenesisConfig { - fn build_storage(mut self) -> ::std::result::Result { + fn build_raw_storage(mut self) -> ::std::result::Result { use std::collections::HashMap; use codec::Encode; diff --git a/polkadot/runtime/wasm/Cargo.lock b/polkadot/runtime/wasm/Cargo.lock index b90d8e14e89d8..6626dd76017e3 100644 --- a/polkadot/runtime/wasm/Cargo.lock +++ b/polkadot/runtime/wasm/Cargo.lock @@ -923,7 +923,6 @@ dependencies = [ "serde 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.64 (registry+https://github.com/rust-lang/crates.io-index)", "substrate-codec 0.1.0", - "substrate-primitives 0.1.0", "substrate-runtime-io 0.1.0", "substrate-runtime-primitives 0.1.0", "substrate-runtime-std 0.1.0", @@ -970,7 +969,7 @@ dependencies = [ "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", - "substrate-primitives 0.1.0", + "substrate-codec 0.1.0", "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm b/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm index 72aa4463a3ed7..0923d033d6029 100644 Binary files a/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm and b/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.compact.wasm differ diff --git a/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.wasm b/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.wasm index c50030ecaadf3..db5b43d0909aa 100755 Binary files a/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.wasm and b/polkadot/runtime/wasm/target/wasm32-unknown-unknown/release/polkadot_runtime.wasm differ diff --git a/substrate/client/src/call_executor.rs b/substrate/client/src/call_executor.rs index ac561b5a549a3..a5e3b712da90c 100644 --- a/substrate/client/src/call_executor.rs +++ b/substrate/client/src/call_executor.rs @@ -132,7 +132,9 @@ impl CallExecutor for LocalCallExecutor let mut overlay = OverlayedChanges::default(); let state = self.backend.state_at(*id)?; let mut externalities = Ext::new(&mut overlay, &state); - let code = externalities.storage(b":code").ok_or(error::ErrorKind::VersionInvalid)? + let code = externalities + .stripped_storage(b":code") + .ok_or(error::ErrorKind::VersionInvalid)? .to_vec(); self.executor.runtime_version(&mut externalities, &code) diff --git a/substrate/executor/src/wasm_executor.rs b/substrate/executor/src/wasm_executor.rs index 4582e423b4f43..b6b6c3faa243c 100644 --- a/substrate/executor/src/wasm_executor.rs +++ b/substrate/executor/src/wasm_executor.rs @@ -222,15 +222,35 @@ impl_function_executor!(this: FunctionExecutor<'e, E>, let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_exists_storage"))?; Ok(if this.ext.exists_storage(&key) { 1 } else { 0 }) }, + ext_exists_previous_storage(key_data: *const u8, key_len: u32) -> u32 => { + let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_exists_previous_storage"))?; + Ok(if this.ext.exists_previous_storage(&key) { 1 } else { 0 }) + }, ext_clear_prefix(prefix_data: *const u8, prefix_len: u32) => { let prefix = this.memory.get(prefix_data, prefix_len as usize).map_err(|_| UserError("Invalid attempt to determine prefix in ext_clear_prefix"))?; this.ext.clear_prefix(&prefix); Ok(()) }, + ext_set_prefix(include_new_keys: u32, prefix_data: *const u8, prefix_len: u32, value_data: *const u8, value_len: u32) => { + let prefix = this.memory.get(prefix_data, prefix_len as usize).map_err(|_| UserError("Invalid attempt to determine prefix in ext_clear_prefix"))?; + let value = this.memory.get(value_data, value_len as usize).map_err(|_| UserError("Invalid attempt to determine value in ext_clear_prefix"))?; + this.ext.set_prefix(include_new_keys != 0, &prefix, value); + Ok(()) + }, + ext_save_prefix_keys(include_new_keys: u32, prefix_data: *const u8, prefix_len: u32, set_prefix_data: *const u8, set_prefix_len: u32) => { + let prefix = this.memory.get(prefix_data, prefix_len as usize).map_err(|_| UserError("Invalid attempt to determine prefix in ext_save_prefix_keys"))?; + let set_prefix = this.memory.get(set_prefix_data, set_prefix_len as usize).map_err(|_| UserError("Invalid attempt to determine set_prefix in ext_save_prefix_keys"))?; + this.ext.save_pefix_keys(include_new_keys != 0, &prefix, &set_prefix); + Ok(()) + }, // return 0 and place u32::max_value() into written_out if no value exists for the key. - ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8 => { + ext_get_allocated_storage(cache_value: u32, key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8 => { let key = this.memory.get(key_data, key_len as usize).map_err(|_| UserError("Invalid attempt to determine key in ext_get_allocated_storage"))?; - let maybe_value = this.ext.storage(&key); + let maybe_value = if cache_value != 0 { + this.ext.cached_storage(&key) + } else { + this.ext.storage(&key) + }; debug_trace!(target: "wasm-trace", "*** Getting storage: {} == {} [k={}]", if let Some(_preimage) = this.hash_lookup.get(&key) { diff --git a/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm b/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm index 790eb1909f21d..4ed55602243df 100644 Binary files a/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm and b/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.compact.wasm differ diff --git a/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.wasm b/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.wasm index 51a27fe0197ad..1a68b54bc2c10 100755 Binary files a/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.wasm and b/substrate/executor/wasm/target/wasm32-unknown-unknown/release/runtime_test.wasm differ diff --git a/substrate/runtime-io/src/keys_set.rs b/substrate/runtime-io/src/keys_set.rs new file mode 100644 index 0000000000000..111d1b3f2b2e6 --- /dev/null +++ b/substrate/runtime-io/src/keys_set.rs @@ -0,0 +1,130 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate Demo. + +// Substrate Demo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate Demo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate Demo. If not, see . + +/// 2nd level prefix for the length of list of set items. +const LIST_LEN_KEY_PREFIX: &'static [u8] = b":list:len"; +/// 2nd level prefix for the items of list of set items. +const LIST_ITEM_KEY_PREFIX: &'static [u8] = b":list:item"; +/// 2nd level prefix for the map of set items. +const MAP_ITEM_PREFIX: &'static [u8] = b":map:"; + +/// Keys set storage. Should write and read raw values (without adding/stripping the prefix). +pub trait Storage { + /// Read `key` from the storage. + fn read(&self, key: &[u8]) -> Option>; + /// Set value under `key` to the storage. + fn set(&mut self, key: &[u8], value: &[u8]); + /// Clear value under `key`. + fn clear(&mut self, key: &[u8]); +} + +/// A set of keys. Holds unique set of storage keys. +/// Allows to enumerate + optionally remove filtered keys (Vec::retain sematics). +pub struct Set<'a, S: 'a> { + /// Prefix for this map. + prefix: &'a [u8], + /// Set storage + storage: &'a mut S, +} + +impl<'a, S: 'a + Storage> Set<'a, S> { + /// Initialize with given prefix. + pub fn new(prefix: &'a [u8], storage: &'a mut S) -> Self { + Set { prefix, storage } + } + + /// Insert item into the set. + pub fn insert(&mut self, item: &[u8]) { + // check if value is already in the set + let map_item_key = compose_key3(self.prefix, MAP_ITEM_PREFIX, item); + match self.storage.read(&map_item_key) { + Some(_) => return, + None => self.storage.set(&map_item_key, &[1]), + } + + // increase list length + let list_len_key = compose_key2(self.prefix, LIST_LEN_KEY_PREFIX); + let list_len: u32 = self.storage.read(&list_len_key) + .and_then(|len| Decode::decode(&mut &len[..])) + .unwrap_or(0); + self.storage.set(&list_len_key, &(list_len + 1).encode()); + + // set item at list length + let list_item_key = compose_key3(self.prefix, LIST_ITEM_KEY_PREFIX, &list_len.encode()); + self.storage.set(&list_item_key, item); + } + + /// Retains only the elements specified by the predicate, which takes key, prefix and value. + /// Returns number of deleted elements + #[allow(dead_code)] // part of the Set, but used only by runtime => allow dead code to keep it together + pub fn retain bool>(&mut self, mut f: F) -> usize { + // read list length + let list_len_key = compose_key2(self.prefix, LIST_LEN_KEY_PREFIX); + let mut list_len: u32 = self.storage.read(&list_len_key) + .and_then(|len| Decode::decode(&mut &len[..])) + .unwrap_or(0); + let original_list_len = list_len; + + // for each list item: run f and remove item if required + let mut idx = 0u32; + while idx < list_len { + let list_item_key = compose_key3(self.prefix, LIST_ITEM_KEY_PREFIX, &idx.encode()); + let list_item = self.storage.read(&list_item_key) + .expect("idx < list_len; list items from 0 to list_len should exist; qed"); + if f(self.storage, &list_item) { + idx += 1; + continue; + } + + // swap_remove item from the list + if idx != list_len - 1 { + let last_list_item_key = compose_key3(self.prefix, LIST_ITEM_KEY_PREFIX, &(list_len - 1).encode()); + let last_list_item = self.storage.read(&last_list_item_key) + .expect("list_len - 1 < list_len; list items from 0 to list_len should exist; qed"); + self.storage.set(&list_item_key, &last_list_item) + } + + // decrease list length + list_len -= 1; + + // clear entry for item in the map + let map_item_key = compose_key3(self.prefix, MAP_ITEM_PREFIX, &list_item); + self.storage.clear(&map_item_key); + } + + self.storage.set(&list_len_key, &list_len.encode()); + + (original_list_len - list_len) as usize + } +} + +/// Compose key of the prefix and the key itself. +fn compose_key2(prefix: &[u8], key: &[u8]) -> Vec { + let mut prefixed_key = prefix.to_vec(); + prefixed_key.extend(key); + prefixed_key +} + +/// Get composite key. +fn compose_key3(prefix1: &[u8], prefix2: &[u8], key: &[u8]) -> Vec { + let mut prefixed_key = prefix1.to_vec(); + prefixed_key.extend(prefix2); + prefixed_key.extend(key); + prefixed_key +} + +// This file is compiled by both susbtrate && runtime => to avoid duplicate tests execution: +// see tests in prefix.rs diff --git a/substrate/runtime-io/src/lib.rs b/substrate/runtime-io/src/lib.rs index 9d57f537f73d8..9e4e9173b8c7f 100644 --- a/substrate/runtime-io/src/lib.rs +++ b/substrate/runtime-io/src/lib.rs @@ -26,6 +26,10 @@ #![cfg_attr(feature = "std", doc = "Substrate runtime standard library as compiled when linked with Rust's standard library.")] #![cfg_attr(not(feature = "std"), doc = "Substrate's runtime standard library as compiled without Rust's standard library.")] +mod prefix; + +pub use prefix::{PREFIX_KEY, PREFIX_LEN_KEY, PurgeFilterResult}; + #[cfg(feature = "std")] include!("../with_std.rs"); diff --git a/substrate/runtime-io/src/prefix.rs b/substrate/runtime-io/src/prefix.rs new file mode 100644 index 0000000000000..825c4e81fcb66 --- /dev/null +++ b/substrate/runtime-io/src/prefix.rs @@ -0,0 +1,350 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +//! Storage value prefix related functions. + +/// The storage entry at this key contains the prefix, which is prepended to all +/// storage values that we're storing. +pub const PREFIX_KEY: &'static [u8] = b":prefix"; +/// The storage prefix of the set of deleted keys. When storage value +/// prefixes are enabled, values are not deleted immediately, but instead marked as +/// deleted (inserted into this set). +/// Runtime could perform purge at some intervals by calling purge_storage(). +pub const DELETED_SET_KEY_PREFIX: &'static [u8] = b":deleted"; + +pub use self::shared::PREFIX_LEN_KEY; +pub use self::api::{is_prefix_configured, read_prefix_len, add_prefix, + strip_prefix, purge, schedule_purge}; + +/// Result of purge filter function. +pub enum PurgeFilterResult { + /// Remove key from both deleted set and storage. + Purge, + /// Remove key from set only. + RemoveFromSet, + /// Leave key in set and in the storage. + LeaveAsIs, +} + +#[cfg(feature = "std")] +mod api { + use substrate_state_machine::Externalities; + use super::keys_set::{Set, Storage as SetStorage}; + use super::{PREFIX_LEN_KEY, PREFIX_KEY, DELETED_SET_KEY_PREFIX, shared}; + use {PurgeFilterResult, raw}; + + struct RawSetStorage<'a>(&'a mut Externalities); + + impl<'a> SetStorage for RawSetStorage<'a> { + fn read(&self, key: &[u8]) -> Option> { raw::storage(self.0, key) } + fn set(&mut self, key: &[u8], value: &[u8]) { raw::set_storage(self.0, key, value); } + fn clear(&mut self, key: &[u8]) { raw::clear_storage(self.0, key); } + } + + /// Returns true if storage value prefix is configured in this runtime. + pub fn is_prefix_configured(ext: &Externalities) -> bool { + raw::cached_storage(ext, PREFIX_LEN_KEY).is_some() + } + + /// Read storage value at the PREFIX_LEN_KEY. + pub fn read_prefix_len(ext: &Externalities) -> Option { + shared::parse_prefix_len(raw::cached_storage(ext, PREFIX_LEN_KEY)) + .expect("Externalities failure is intercepted by executor") + } + + /// Add prefix to the value. + pub fn add_prefix(ext: &Externalities, value: &[u8]) -> Vec { + shared::add_prefix( + raw::cached_storage(ext, PREFIX_KEY), + read_prefix_len(ext), + value) + } + + /// Strip prefix from the value. + pub fn strip_prefix(ext: &Externalities, value: Vec) -> (Option>, Option>) { + shared::strip_prefix(raw::cached_storage(ext, PREFIX_LEN_KEY), value) + .expect("Externalities failure is intercepted by executor") + } + + /// Purge scheduled values. + pub fn purge>, Option>) -> PurgeFilterResult>(ext: &mut Externalities, f: F) -> usize { + let prefix_len = raw::cached_storage(ext, PREFIX_LEN_KEY); + Set::new(DELETED_SET_KEY_PREFIX, &mut RawSetStorage(ext)).retain(|storage, key| { + let (prefix, value) = storage.read(key) + .map(|value| shared::strip_prefix(prefix_len.clone(), value) + .expect("Externalities failure is intercepted by executor")) + .unwrap_or_default(); + match f(key, prefix, value) { + PurgeFilterResult::Purge => { + storage.clear(key); + false + }, + PurgeFilterResult::RemoveFromSet => false, + PurgeFilterResult::LeaveAsIs => true, + } + }) + } + + /// Schedule value at given `key` for future purge. + pub fn schedule_purge(ext: &mut Externalities, key: &[u8]) { + Set::new(DELETED_SET_KEY_PREFIX, &mut RawSetStorage(ext)).insert(key); + } +} + +#[cfg(not(feature = "std"))] +mod api { + use rstd::vec::Vec; + use super::keys_set::{Set, Storage as SetStorage}; + use super::{PREFIX_LEN_KEY, PREFIX_KEY, DELETED_SET_KEY_PREFIX, shared}; + use {PurgeFilterResult, raw}; + + struct RawSetStorage; + + impl SetStorage for RawSetStorage { + fn read(&self, key: &[u8]) -> Option> { raw::storage(key) } + fn set(&mut self, key: &[u8], value: &[u8]) { raw::set_storage(key, value); } + fn clear(&mut self, key: &[u8]) { raw::clear_storage(key); } + } + + /// Returns true if storage value prefix is configured in this runtime. + pub fn is_prefix_configured() -> bool { + raw::cached_storage(PREFIX_LEN_KEY).is_some() + } + + /// Read storage value at the PREFIX_LEN_KEY. + pub fn read_prefix_len() -> Option { + shared::parse_prefix_len(raw::cached_storage(PREFIX_LEN_KEY)) + .expect("Externalities failure is intercepted by executor") + } + + /// Add prefix to the value. + pub fn add_prefix(value: &[u8]) -> Vec { + shared::add_prefix( + raw::cached_storage(PREFIX_KEY), + read_prefix_len(), + value) + } + + /// Strip prefix from the value. + pub fn strip_prefix(value: Vec) -> (Option>, Option>) { + shared::strip_prefix(raw::cached_storage(PREFIX_LEN_KEY), value) + .expect("Externalities failure is intercepted by executor") + } + + /// Purge scheduled values. + pub fn purge>, Option>) -> PurgeFilterResult>(f: F) -> usize { + let prefix_len = raw::cached_storage(PREFIX_LEN_KEY); + Set::new(DELETED_SET_KEY_PREFIX, &mut RawSetStorage).retain(|storage, key| { + let (prefix, value) = storage.read(key) + .map(|value| shared::strip_prefix(prefix_len.clone(), value) + .expect("Externalities failure is intercepted by executor")) + .unwrap_or_default(); + match f(key, prefix, value) { + PurgeFilterResult::Purge => { + storage.clear(key); + false + }, + PurgeFilterResult::RemoveFromSet => false, + PurgeFilterResult::LeaveAsIs => true, + } + }) + } + + /// Schedule value at given `key` for future purge. + pub fn schedule_purge(key: &[u8]) { + Set::new(DELETED_SET_KEY_PREFIX, &mut RawSetStorage).insert(key); + } +} + +mod shared { + #[cfg(not(feature = "std"))] + use rstd::vec::Vec; + + pub fn add_prefix(prefix_value: Option>, prefix_len: Option, value: &[u8]) -> Vec { + assert_eq!(prefix_value.is_some(), prefix_len.is_some()); + + prefix_value + .map(|prefix| { + let prefix_len = prefix_len.expect("asserted above") as usize; + let mut prefixed_value = prefix[..prefix_len].to_vec(); + prefixed_value.extend(value); + prefixed_value + }) + .unwrap_or_else(|| value.to_vec()) + } + + include!("./prefix_shared.rs"); +} + +pub mod keys_set { + #[cfg(not(feature = "std"))] + use rstd::vec::Vec; + use codec::{Decode, Encode}; + + include!("./keys_set.rs"); +} + + +#[cfg(all(feature = "std", test))] +mod tests { + use std::collections::BTreeSet; + use codec::Encode; + use substrate_state_machine::{Externalities, TestExternalities}; + use super::{PREFIX_KEY, PREFIX_LEN_KEY, is_prefix_configured, + read_prefix_len, add_prefix, strip_prefix}; + use super::shared::parse_prefix_len; + use super::keys_set::{Set as KeysSet, Storage as KeysSetStorage}; + + impl<'a> KeysSetStorage for &'a mut TestExternalities { + fn read(&self, key: &[u8]) -> Option> { self.storage(key) } + fn set(&mut self, key: &[u8], value: &[u8]) { self.place_storage(key.to_vec(), Some(value.to_vec())); } + fn clear(&mut self, key: &[u8]) { self.place_storage(key.to_vec(), None); } + } + + #[test] + fn is_prefix_configured_works() { + let mut ext = TestExternalities::default(); + assert!(!is_prefix_configured(&ext)); + ext.set_storage(PREFIX_LEN_KEY.to_vec(), 4u64.encode()); + assert!(is_prefix_configured(&ext)); + } + + #[test] + fn read_prefix_len_works() { + let mut ext = TestExternalities::default(); + assert_eq!(read_prefix_len(&ext), None); + ext.set_storage(PREFIX_LEN_KEY.to_vec(), vec![0, 0, 0, 0, 4, 0, 0, 0]); + assert_eq!(read_prefix_len(&ext), Some(4)); + } + + #[test] + fn add_prefix_works() { + let mut ext = TestExternalities::default(); + assert_eq!(add_prefix(&ext, &[42]), vec![42]); + ext.set_storage(PREFIX_LEN_KEY.to_vec(), vec![0, 0, 0, 3, 0, 0, 0]); + ext.set_storage(PREFIX_KEY.to_vec(), vec![1, 2, 3]); + assert_eq!(read_prefix_len(&ext), Some(3)); + assert_eq!(add_prefix(&ext, &[42]), vec![1, 2, 3, 42]); + } + + #[test] + fn strip_prefix_works() { + let mut ext = TestExternalities::default(); + assert_eq!(strip_prefix(&ext, vec![1, 2, 3, 42]), (None, Some(vec![1, 2, 3, 42]))); + ext.set_storage(PREFIX_LEN_KEY.to_vec(), vec![0, 0, 0, 3, 0, 0, 0]); + assert_eq!(strip_prefix(&ext, vec![1, 2, 3, 42]), (Some(vec![1, 2, 3]), Some(vec![42]))); + assert_eq!(strip_prefix(&ext, vec![1, 2, 3]), (Some(vec![1, 2, 3]), None)); + } + + #[test] + fn parse_prefix_len_succeds() { + assert_eq!(parse_prefix_len(None), Ok(None)); + assert_eq!(parse_prefix_len(Some(vec![])), Ok(None)); + assert_eq!(parse_prefix_len(Some(vec![0, 0, 0, 0])), Ok(None)); + assert_eq!(parse_prefix_len(Some(vec![0, 0, 0, 0, 4, 0, 0, 0])), Ok(Some(4))); + assert_eq!(parse_prefix_len(Some(vec![0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0])), Ok(Some(8))); + assert_eq!(parse_prefix_len(Some(vec![5, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0])), Ok(Some(8))); + } + + #[test] + fn parse_prefix_len_fails() { + assert!(parse_prefix_len(Some(vec![0])).is_err()); + assert!(parse_prefix_len(Some(vec![0, 0, 0])).is_err()); + assert!(parse_prefix_len(Some(vec![0, 0, 4, 0, 0, 0])).is_err()); + } + + fn with_keys_set)>(mut f: F) -> Vec> { + let mut ext = TestExternalities::default(); + let mut storage = &mut ext; + let mut set = KeysSet::new(b"set", &mut storage); + f(&mut set); + + let mut keys = BTreeSet::new(); + set.retain(|_, key| { + keys.insert(key.to_vec()); + false + }); + + keys.into_iter().collect() + } + + #[test] + fn keys_are_inserted_into_keys_set() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key2"); + set.insert(b"key3"); + }), vec![b"key1".to_vec(), b"key2".to_vec(), b"key3".to_vec()]); + } + + #[test] + fn keys_are_retained_from_the_middle_of_keys_set() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key2"); + set.insert(b"key3"); + set.retain(|_, key| key != b"key2"); + }), vec![b"key1".to_vec(), b"key3".to_vec()]); + } + + #[test] + fn keys_are_retained_from_the_beginning_of_keys_set() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key2"); + set.insert(b"key3"); + set.retain(|_, key| key != b"key1"); + }), vec![b"key2".to_vec(), b"key3".to_vec()]); + } + + #[test] + fn keys_are_retained_from_the_end_of_keys_set() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key2"); + set.insert(b"key3"); + set.retain(|_, key| key != b"key3"); + }), vec![b"key1".to_vec(), b"key2".to_vec()]); + } + + #[test] + fn keys_are_fully_retained_from_keys_set() { + assert!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key2"); + set.insert(b"key3"); + set.retain(|_, _| false); + }).is_empty()); + } + + #[test] + fn duplicate_keys_are_not_inserted_into_keys_set() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.insert(b"key1"); + }), vec![b"key1".to_vec()]); + } + + #[test] + fn keys_are_inserted_after_removal() { + assert_eq!(with_keys_set(|set| { + set.insert(b"key1"); + set.retain(|_, _| false); + set.insert(b"key1"); + set.insert(b"key2"); + }), vec![b"key1".to_vec(), b"key2".to_vec()]); + } +} diff --git a/substrate/runtime-io/src/prefix_shared.rs b/substrate/runtime-io/src/prefix_shared.rs new file mode 100644 index 0000000000000..6e3053ddfc4c0 --- /dev/null +++ b/substrate/runtime-io/src/prefix_shared.rs @@ -0,0 +1,70 @@ +// Copyright 2018 Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use codec::Decode; + +/// Prefix length storage key. +pub const PREFIX_LEN_KEY: &'static [u8] = b":prefix_len"; + +/// Strips the prefix from the storage value. +pub fn strip_prefix(prefix_len_value: Option>, value: Vec) -> Result<(Option>, Option>), &'static str> { + let value_len = value.len(); + match parse_prefix_len(prefix_len_value)? { + None => Ok((None, Some(value))), + Some(prefix_len) if value_len < prefix_len as usize => + Err("Too short storage value to have prefix"), + Some(prefix_len) if value_len == prefix_len as usize => + Ok((Some(value), None)), + Some(prefix_len) => { + let mut prefix = value; + let value = prefix.split_off(prefix_len as usize); + Ok((Some(prefix), Some(value))) + }, + } +} + +/// Parse prefix length value +pub fn parse_prefix_len(value: Option>) -> Result, &'static str> { + // prefix lenght is prefixed itself, but we do not know the length here + // => the length is encoded as u32 and it is stored after the prefix + // => read the u32 from the end and then check if the value is of correct length + let value = match value { + Some(value) => value, + None => return Ok(None), + }; + + let value_len = value.len(); + match value_len { + 0 => Ok(None), + _ if value_len < 4 => Err("Too short prefix length storage value"), + _ => { + let prefix_len: u32 = Decode::decode(&mut &value[value_len - 4..]) + .expect("passed 4-bytes slice to decode; 4 bytes are required to decode u32; qed"); + if value_len == prefix_len as usize + 4 { + if prefix_len == 0 { + Ok(None) + } else { + Ok(Some(prefix_len)) + } + } else { + Err("Invalid prefix length storage value") + } + } + } +} + +// This file is compiled by both susbtrate && runtime => to avoid duplicate tests execution: +// see tests in prefix.rs diff --git a/substrate/runtime-io/with_std.rs b/substrate/runtime-io/with_std.rs index 3d3b2552082b3..7f382ee3f7549 100644 --- a/substrate/runtime-io/with_std.rs +++ b/substrate/runtime-io/with_std.rs @@ -31,55 +31,121 @@ pub use primitives::{blake2_256, twox_128, twox_256}; pub use substrate_state_machine::{Externalities, TestExternalities}; use primitives::hexdisplay::HexDisplay; +use prefix::{DELETED_SET_KEY_PREFIX, is_prefix_configured, add_prefix, + strip_prefix, read_prefix_len, purge, schedule_purge}; // TODO: use the real error, not NoError. environmental!(ext: trait Externalities); +/// Set storage values prefix. +pub fn set_storage_prefix(prefix: &[u8]) { + ext::with(|ext| { + // both branches in following match could be implemented later, but it will involve + // upgrading the whole storage + match read_prefix_len(ext) { + // do nothing if we're not configured to work with prefix + None => return, + // check that new prefix length is the same as we configured for + Some(existing_prefix_len) => assert_eq!( + prefix.len(), + existing_prefix_len as usize, + "Change of prefix length is not currently supported"), + }; + + let mut prefixed_prefix = prefix.to_vec(); + prefixed_prefix.extend_from_slice(prefix); + + ext.set_storage(PREFIX_KEY.to_vec(), prefixed_prefix) + }); +} + +/// Purge scheduled entries from the storage. +pub fn purge_storage>, Option>) -> PurgeFilterResult>(f: F) -> usize { + ext::with(|ext| { + if is_prefix_configured(ext) { + purge(ext, f) + } else { + 0 + } + }).unwrap_or_default() +} + /// Get `key` from storage and return a `Vec`, empty if there's a problem. pub fn storage(key: &[u8]) -> Option> { - ext::with(|ext| ext.storage(key).map(|s| s.to_vec())) - .expect("read_storage cannot be called outside of an Externalities-provided environment.") + ext::with(|ext| ext.storage(key) + .and_then(|v| strip_prefix(ext, v).1)) + .expect("storage cannot be called outside of an Externalities-provided environment.") +} + +/// Get `key` prefix from storage and return a `Vec`, empty if storage prefix is not configured. +pub fn storage_prefix(key: &[u8]) -> Option> { + ext::with(|ext| ext.storage(key) + .and_then(|v| strip_prefix(ext, v).0)) + .expect("storage_prefix cannot be called outside of an Externalities-provided environment.") } /// Get `key` from storage, placing the value into `value_out` (as much as possible) and return /// the number of bytes that the key in storage was beyond the offset or None if the storage entry /// doesn't exist at all. pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { - ext::with(|ext| ext.storage(key).map(|value| { - let value = &value[value_offset..]; - let written = ::std::cmp::min(value.len(), value_out.len()); - value_out[0..written].copy_from_slice(&value[0..written]); - value.len() - })).expect("read_storage cannot be called outside of an Externalities-provided environment.") + ext::with(|ext| ext.storage(key) + .and_then(|v| strip_prefix(ext, v).1) + .map(|value| { + let value = &value[value_offset..]; + let written = ::std::cmp::min(value.len(), value_out.len()); + value_out[0..written].copy_from_slice(&value[0..written]); + value.len() + })).expect("read_storage cannot be called outside of an Externalities-provided environment.") } /// Set the storage of some particular key to Some value. pub fn set_storage(key: &[u8], value: &[u8]) { - ext::with(|ext| - ext.set_storage(key.to_vec(), value.to_vec()) - ); + ext::with(|ext| { + let prefixed_value = add_prefix(ext, value); + ext.set_storage(key.to_vec(), prefixed_value); + }); } /// Clear the storage of some particular key. pub fn clear_storage(key: &[u8]) { - ext::with(|ext| - ext.clear_storage(key) - ); + ext::with(|ext| { + let empty_value = add_prefix(ext, &[]); + // if we do not use prefix OR value has been created + deleted in the same block => just remove + if empty_value.is_empty() || !ext.exists_previous_storage(key) { + ext.clear_storage(key) + } else { + schedule_purge(ext, key); + ext.set_storage(key.to_vec(), empty_value) + } + }); } /// Check whether a given `key` exists in storage. pub fn exists_storage(key: &[u8]) -> bool { ext::with(|ext| - ext.exists_storage(key) + if !is_prefix_configured(ext) { + ext.exists_storage(key) + } else { + ext.storage(key) + .map(|v| strip_prefix(ext, v).1.is_some()) + .unwrap_or(false) + } + ).unwrap_or(false) } /// Clear the storage entries key of which starts with the given prefix. pub fn clear_prefix(prefix: &[u8]) { - ext::with(|ext| - ext.clear_prefix(prefix) - ); + ext::with(|ext| { + let empty_value = add_prefix(ext, &[]); + if empty_value.is_empty() { + ext.clear_prefix(prefix) + } else { + ext.save_pefix_keys(false, prefix, DELETED_SET_KEY_PREFIX); + ext.set_prefix(false, prefix, empty_value) + } + }); } /// The current relay chain identifier. @@ -157,6 +223,30 @@ pub fn print(value: T) { value.print(); } +pub(crate) mod raw { + use substrate_state_machine::Externalities; + + /// Get raw `key` from storage and return a `Vec`, empty if there's a problem. + pub fn storage(ext: &Externalities, key: &[u8]) -> Option> { + ext.storage(key) + } + + /// Get cached raw `key` from storage and return a `Vec`, empty if there's a problem. + pub fn cached_storage(ext: &Externalities, key: &[u8]) -> Option> { + ext.cached_storage(key) + } + + /// Set the storage of some particular key to Some value. + pub fn set_storage(ext: &mut Externalities, key: &[u8], value: &[u8]) { + ext.set_storage(key.to_vec(), value.to_vec()); + } + + /// Clear the storage of some particular key. + pub fn clear_storage(ext: &mut Externalities, key: &[u8]) { + ext.clear_storage(key) + } +} + #[macro_export] macro_rules! impl_stubs { ( $( $new_name:ident $($nodecode:ident)* => $invoke: expr ),*) => { @@ -191,7 +281,7 @@ mod std_tests { #[test] fn storage_works() { - let mut t = TestExternalities::new(); + let mut t = TestExternalities::default(); assert!(with_externalities(&mut t, || { assert_eq!(storage(b"hello"), None); set_storage(b"hello", b"world"); diff --git a/substrate/runtime-io/without_std.rs b/substrate/runtime-io/without_std.rs index 438b1a84fc976..09c9e54d54179 100644 --- a/substrate/runtime-io/without_std.rs +++ b/substrate/runtime-io/without_std.rs @@ -26,6 +26,8 @@ pub extern crate substrate_codec as codec; use core::intrinsics; use rstd::vec::Vec; pub use rstd::{mem, slice}; +use prefix::{DELETED_SET_KEY_PREFIX, is_prefix_configured, add_prefix, + strip_prefix, read_prefix_len, purge, schedule_purge}; #[panic_implementation] #[no_mangle] @@ -57,8 +59,11 @@ extern "C" { fn ext_set_storage(key_data: *const u8, key_len: u32, value_data: *const u8, value_len: u32); fn ext_clear_storage(key_data: *const u8, key_len: u32); fn ext_exists_storage(key_data: *const u8, key_len: u32) -> u32; + fn ext_exists_previous_storage(key_data: *const u8, key_len: u32) -> u32; fn ext_clear_prefix(prefix_data: *const u8, prefix_len: u32); - fn ext_get_allocated_storage(key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8; + fn ext_set_prefix(include_new_keys: u32, prefix_data: *const u8, prefix_len: u32, value_data: *const u8, value_len: u32); + fn ext_save_prefix_keys(include_new_keys: u32, prefix_data: *const u8, prefix_len: u32, set_prefix_data: *const u8, set_prefix_len: u32); + fn ext_get_allocated_storage(cache_value: u32, key_data: *const u8, key_len: u32, written_out: *mut u32) -> *mut u8; fn ext_get_storage_into(key_data: *const u8, key_len: u32, value_data: *mut u8, value_len: u32, value_offset: u32) -> u32; fn ext_storage_root(result: *mut u8); fn ext_enumerated_trie_root(values_data: *const u8, lens_data: *const u32, lens_len: u32, result: *mut u8); @@ -69,60 +74,114 @@ extern "C" { fn ext_ed25519_verify(msg_data: *const u8, msg_len: u32, sig_data: *const u8, pubkey_data: *const u8) -> u32; } -/// Get `key` from storage and return a `Vec`, empty if there's a problem. -pub fn storage(key: &[u8]) -> Option> { - let mut length: u32 = 0; +/// Set storage values prefix. +pub fn set_storage_prefix(prefix: &[u8]) { + // both branches in following match could be implemented later, but it will involve + // upgrading the whole storage + match read_prefix_len() { + // do nothing if we're not configured to work with prefix + None => return, + // check that new prefix length is the same as we configured for + Some(existing_prefix_len) => assert_eq!( + prefix.len(), + existing_prefix_len as usize, + "Change of prefix length is not currently supported"), + }; + + let mut prefixed_prefix = prefix.to_vec(); + prefixed_prefix.extend_from_slice(prefix); unsafe { - let ptr = ext_get_allocated_storage(key.as_ptr(), key.len() as u32, &mut length); - if length == u32::max_value() { - None - } else { - Some(Vec::from_raw_parts(ptr, length as usize, length as usize)) - } + ext_set_storage( + PREFIX_KEY.as_ptr(), PREFIX_KEY.len() as u32, + prefixed_prefix.as_ptr(), prefixed_prefix.len() as u32 + ); + } +} + +/// Purge scheduled entries from the storage. +pub fn purge_storage>, Option>) -> PurgeFilterResult>(f: F) -> usize { + if is_prefix_configured() { + purge(f) + } else { + 0 } } +/// Get `key` from storage and return a `Vec`, empty if there's a problem. +pub fn storage(key: &[u8]) -> Option> { + self::raw::storage(key).and_then(|v| strip_prefix(v).1) +} + +/// Get `key` prefix from storage and return a `Vec`, empty if storage prefix is not configured. +pub fn storage_prefix(key: &[u8]) -> Option> { + self::raw::storage(key).and_then(|v| strip_prefix(v).0) +} + /// Set the storage of some particular key to Some value. pub fn set_storage(key: &[u8], value: &[u8]) { - unsafe { - ext_set_storage( - key.as_ptr(), key.len() as u32, - value.as_ptr(), value.len() as u32 - ); - } + self::raw::set_storage(key, &add_prefix(value)) } /// Clear the storage of some particular key. pub fn clear_storage(key: &[u8]) { + let empty_value = add_prefix(&[]); unsafe { - ext_clear_storage( - key.as_ptr(), key.len() as u32 - ); + // if we do not use prefix OR value has been created + deleted in the same block => just remove + if empty_value.is_empty() || ext_exists_previous_storage(key.as_ptr(), key.len() as u32) == 0 { + ext_clear_storage( + key.as_ptr(), key.len() as u32 + ); + } else { + schedule_purge(key); + ext_set_storage( + key.as_ptr(), key.len() as u32, + empty_value.as_ptr(), empty_value.len() as u32 + ); + } } } /// Determine whether a particular key exists in storage. pub fn exists_storage(key: &[u8]) -> bool { unsafe { - ext_exists_storage( - key.as_ptr(), key.len() as u32 - ) != 0 + if !is_prefix_configured() { + ext_exists_storage( + key.as_ptr(), key.len() as u32 + ) != 0 + } else { + storage(key).is_some() + } } } /// Clear the storage entries key of which starts with the given prefix. pub fn clear_prefix(prefix: &[u8]) { + let empty_value = add_prefix(&[]); unsafe { - ext_clear_prefix( - prefix.as_ptr(), - prefix.len() as u32 - ); + if empty_value.is_empty() { + ext_clear_prefix( + prefix.as_ptr(), + prefix.len() as u32 + ); + } else { + ext_save_prefix_keys( + 0, + prefix.as_ptr(), prefix.len() as u32, + DELETED_SET_KEY_PREFIX.as_ptr(), DELETED_SET_KEY_PREFIX.len() as u32); + ext_set_prefix( + 0, + prefix.as_ptr(), prefix.len() as u32, + empty_value.as_ptr(), empty_value.len() as u32 + ); + } } } /// Get `key` from storage, placing the value into `value_out` (as much as possible) and return /// the number of bytes that the key in storage was beyond the offset. pub fn read_storage(key: &[u8], value_out: &mut [u8], value_offset: usize) -> Option { + let prefix_len = read_prefix_len().unwrap_or(0); + let value_offset = value_offset + prefix_len as usize; unsafe { match ext_get_storage_into( key.as_ptr(), key.len() as u32, @@ -253,6 +312,56 @@ pub fn print(value: T) { value.print(); } +pub(crate) mod raw { + use rstd::vec::Vec; + use super::{ext_get_allocated_storage, ext_set_storage, ext_clear_storage}; + + /// Get raw `key` from storage and return a `Vec`, empty if there's a problem. + pub fn storage(key: &[u8]) -> Option> { + let mut length: u32 = 0; + unsafe { + let ptr = ext_get_allocated_storage(0, key.as_ptr(), key.len() as u32, &mut length); + if length == u32::max_value() { + None + } else { + Some(Vec::from_raw_parts(ptr, length as usize, length as usize)) + } + } + } + + /// Get cached raw `key` from storage and return a `Vec`, empty if there's a problem. + pub fn cached_storage(key: &[u8]) -> Option> { + let mut length: u32 = 0; + unsafe { + let ptr = ext_get_allocated_storage(1, key.as_ptr(), key.len() as u32, &mut length); + if length == u32::max_value() { + None + } else { + Some(Vec::from_raw_parts(ptr, length as usize, length as usize)) + } + } + } + + /// Set the storage of some particular key to Some value. + pub fn set_storage(key: &[u8], value: &[u8]) { + unsafe { + ext_set_storage( + key.as_ptr(), key.len() as u32, + value.as_ptr(), value.len() as u32 + ); + } + } + + /// Clear the storage of some particular key. + pub fn clear_storage(key: &[u8]) { + unsafe { + ext_clear_storage( + key.as_ptr(), key.len() as u32 + ); + } + } +} + #[macro_export] macro_rules! impl_stubs { ( $( $new_name:ident $($nodecode:ident)* => $invoke:expr ),* ) => { diff --git a/substrate/runtime-support/src/storage/mod.rs b/substrate/runtime-support/src/storage/mod.rs index 3009b5f36bf1c..97a96dad33166 100644 --- a/substrate/runtime-support/src/storage/mod.rs +++ b/substrate/runtime-support/src/storage/mod.rs @@ -55,7 +55,7 @@ impl<'a, I: Input + 'a> Input for AppendZeroes<'a, I> { } } - /// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. +/// Return the value of the item in storage under `key`, or `None` if there is no explicit entry. pub fn get(key: &[u8]) -> Option { let key = twox_128(key); runtime_io::read_storage(&key[..], &mut [0; 0][..], 0).map(|_| { @@ -67,6 +67,12 @@ pub fn get(key: &[u8]) -> Option { }) } +/// Return the prefix of the value of the item in storage under `key`, or `None` +/// if there is no explicit entry OR there is no prefix. +pub fn get_prefix(key: &[u8]) -> Option> { + runtime_io::storage_prefix(&twox_128(key)[..]) +} + /// Return the value of the item in storage under `key`, or the type's default if there is no /// explicit entry. pub fn get_or_default(key: &[u8]) -> T { diff --git a/substrate/runtime/consensus/src/lib.rs b/substrate/runtime/consensus/src/lib.rs index 13d40afe5eab1..a6a2b8c80f5b7 100644 --- a/substrate/runtime/consensus/src/lib.rs +++ b/substrate/runtime/consensus/src/lib.rs @@ -146,7 +146,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> ::std::result::Result { + fn build_raw_storage(self) -> ::std::result::Result { use codec::{Encode, KeyedVec}; let auth_count = self.authorities.len() as u32; let mut r: runtime_io::TestExternalities = self.authorities.into_iter().enumerate().map(|(i, v)| diff --git a/substrate/runtime/contract/src/genesis_config.rs b/substrate/runtime/contract/src/genesis_config.rs index d841f22806a6a..9e3cf3eaa1eb1 100644 --- a/substrate/runtime/contract/src/genesis_config.rs +++ b/substrate/runtime/contract/src/genesis_config.rs @@ -35,7 +35,7 @@ pub struct GenesisConfig { } impl runtime_primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> Result { + fn build_raw_storage(self) -> Result { let r: runtime_io::TestExternalities = map![ twox_128(>::key()).to_vec() => self.contract_fee.encode(), twox_128(>::key()).to_vec() => self.call_base_fee.encode(), diff --git a/substrate/runtime/council/src/lib.rs b/substrate/runtime/council/src/lib.rs index 56d87ea59bff5..f3b09a6d3c941 100644 --- a/substrate/runtime/council/src/lib.rs +++ b/substrate/runtime/council/src/lib.rs @@ -590,7 +590,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> ::std::result::Result { + fn build_raw_storage(self) -> ::std::result::Result { use codec::Encode; Ok(map![ diff --git a/substrate/runtime/democracy/src/lib.rs b/substrate/runtime/democracy/src/lib.rs index 9f9c00d056ab5..7dfe4f07ac042 100644 --- a/substrate/runtime/democracy/src/lib.rs +++ b/substrate/runtime/democracy/src/lib.rs @@ -338,7 +338,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> ::std::result::Result { + fn build_raw_storage(self) -> ::std::result::Result { use codec::Encode; Ok(map![ diff --git a/substrate/runtime/primitives/src/lib.rs b/substrate/runtime/primitives/src/lib.rs index 01db862f0da89..723dfc5d3ddb1 100644 --- a/substrate/runtime/primitives/src/lib.rs +++ b/substrate/runtime/primitives/src/lib.rs @@ -65,18 +65,31 @@ pub type StorageMap = HashMap, Vec>; /// Complex storage builder stuff. #[cfg(feature = "std")] -pub trait BuildStorage { +pub trait BuildStorage where Self: Sized { fn hash(data: &[u8]) -> [u8; 16] { let r = runtime_io::twox_128(data); trace!(target: "build_storage", "{} <= {}", substrate_primitives::hexdisplay::HexDisplay::from(&r), ascii_format(data)); r } - fn build_storage(self) -> Result; + + /// Build raw storage, where values are stored without any prefix. + fn build_raw_storage(self) -> Result; + + /// Build final storage where values are prefixed with the storage prefix (if configured). + fn build_storage(self) -> Result { + let mut storage = self.build_raw_storage()?; + if let Some(prefix) = storage.get(runtime_io::PREFIX_KEY).cloned() { + for value in storage.values_mut() { + *value = prefix.iter().cloned().chain(value.iter().cloned()).collect(); + } + } + Ok(storage) + } } #[cfg(feature = "std")] impl BuildStorage for StorageMap { - fn build_storage(self) -> Result { + fn build_raw_storage(self) -> Result { Ok(self) } } @@ -272,11 +285,11 @@ macro_rules! impl_outer_config { } #[cfg(any(feature = "std", test))] impl $crate::BuildStorage for $main { - fn build_storage(self) -> ::std::result::Result<$crate::StorageMap, String> { + fn build_raw_storage(self) -> ::std::result::Result<$crate::StorageMap, String> { let mut s = $crate::StorageMap::new(); $( if let Some(extra) = self.$snake { - s.extend(extra.build_storage()?); + s.extend(extra.build_raw_storage()?); } )* Ok(s) diff --git a/substrate/runtime/session/src/lib.rs b/substrate/runtime/session/src/lib.rs index 6e320a73575f9..f213eeb99bc00 100644 --- a/substrate/runtime/session/src/lib.rs +++ b/substrate/runtime/session/src/lib.rs @@ -244,7 +244,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> ::std::result::Result { + fn build_raw_storage(self) -> ::std::result::Result { use codec::Encode; use primitives::traits::As; Ok(map![ diff --git a/substrate/runtime/staking/src/genesis_config.rs b/substrate/runtime/staking/src/genesis_config.rs index 2987725b7a235..dbb6c272f843d 100644 --- a/substrate/runtime/staking/src/genesis_config.rs +++ b/substrate/runtime/staking/src/genesis_config.rs @@ -118,7 +118,7 @@ impl Default for GenesisConfig { } impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> Result { + fn build_raw_storage(self) -> Result { let total_stake: T::Balance = self.balances.iter().fold(Zero::zero(), |acc, &(_, n)| acc + n); let mut r: runtime_io::TestExternalities = map![ diff --git a/substrate/runtime/system/Cargo.toml b/substrate/runtime/system/Cargo.toml index 38fb97b582f5c..c0a6e93abfecf 100644 --- a/substrate/runtime/system/Cargo.toml +++ b/substrate/runtime/system/Cargo.toml @@ -9,12 +9,14 @@ serde = { version = "1.0", default_features = false } serde_derive = { version = "1.0", optional = true } safe-mix = { path = "../../../safe-mix", default_features = false} substrate-codec = { path = "../../codec", default_features = false } -substrate-primitives = { path = "../../primitives", default_features = false } substrate-runtime-std = { path = "../../runtime-std", default_features = false } substrate-runtime-io = { path = "../../runtime-io", default_features = false } substrate-runtime-support = { path = "../../runtime-support", default_features = false } substrate-runtime-primitives = { path = "../primitives", default_features = false } +[dev-dependencies] +substrate-primitives = { path = "../../primitives", default_features = false } + [features] default = ["std"] std = [ @@ -22,7 +24,6 @@ std = [ "serde_derive", "safe-mix/std", "substrate-codec/std", - "substrate-primitives/std", "substrate-runtime-std/std", "substrate-runtime-io/std", "substrate-runtime-support/std", diff --git a/substrate/runtime/system/src/lib.rs b/substrate/runtime/system/src/lib.rs index 002c540590557..c2ee4cc98c7cb 100644 --- a/substrate/runtime/system/src/lib.rs +++ b/substrate/runtime/system/src/lib.rs @@ -37,20 +37,21 @@ extern crate substrate_codec as codec; extern crate substrate_runtime_primitives as primitives; extern crate safe_mix; +#[cfg(test)] +extern crate substrate_primitives; + +use codec::{Decode, Encode}; use rstd::prelude::*; use primitives::traits::{self, CheckEqual, SimpleArithmetic, SimpleBitOps, Zero, One, Bounded, - Hash, Member, MaybeDisplay}; -use runtime_support::{StorageValue, StorageMap, Parameter}; + Hash, Member, MaybeDisplay, As}; +use runtime_support::{storage, StorageValue, StorageMap, Parameter}; use safe_mix::TripletMix; -#[cfg(any(feature = "std", test))] -use rstd::marker::PhantomData; -#[cfg(any(feature = "std", test))] -use codec::Encode; - #[cfg(any(feature = "std", test))] use runtime_io::{twox_128, TestExternalities}; +mod tests; + /// Compute the extrinsics root of a list of extrinsics. pub fn extrinsics_root(extrinsics: &[E]) -> H::Output { extrinsics_data_root::(extrinsics.iter().map(codec::Encode::encode).collect()) @@ -89,6 +90,8 @@ decl_storage! { pub ExtrinsicIndex get(extrinsic_index): b"sys:xti" => required u32; pub ExtrinsicData get(extrinsic_data): b"sys:xtd" => required map [ u32 => Vec ]; RandomSeed get(random_seed): b"sys:rnd" => required T::Hash; + StoragePurgeInterval get(storage_purge_interval): b"sys:purge_interval" => default T::BlockNumber; + MinPurgedValueAge get(min_purged_value_age): b"sys:min_purged_value_age" => default T::BlockNumber; // The current block number being processed. Set by `execute_block`. Number get(block_number): b"sys:num" => required T::BlockNumber; ParentHash get(parent_hash): b"sys:pha" => required T::Hash; @@ -99,7 +102,33 @@ decl_storage! { impl Module { /// Start the execution of a particular block. pub fn initialise(number: &T::BlockNumber, parent_hash: &T::Hash, txs_root: &T::Hash) { - // populate environment. + // set new storage prefix if we're configured to use prefix + runtime_io::set_storage_prefix(&number.as_().encode()); + // ..then purge all scheduled entries if the time has come + let storage_purge_interval = >::get(); + if storage_purge_interval == Zero::zero() || *number % storage_purge_interval == Zero::zero() { + let min_purged_value_age = >::get(); + if *number > min_purged_value_age { + let ancient_block = (*number - min_purged_value_age).as_(); + runtime_io::purge_storage(|_, prefix, value| { + // if key is not deleted => delete from deleted keys set + if value.is_some() { + return runtime_io::PurgeFilterResult::RemoveFromSet; + } + + // if key is deleted later than the ancient_block => leave for another period + if prefix.and_then(|prefix| Decode::decode(&mut &prefix[..])) + .map(|deleted_at_block: u64| deleted_at_block > ancient_block) + .unwrap_or(false) { + return runtime_io::PurgeFilterResult::LeaveAsIs; + } + + runtime_io::PurgeFilterResult::Purge + }); + } + } + + // ..and finally populate environment. >::put(number); >::put(parent_hash); >::insert(*number - One::one(), parent_hash); @@ -128,6 +157,17 @@ impl Module { >::put(l); } + /// Gets the number of block at which given key value has been changed last or None + /// if value is not set OR runtime is NOT configured to store this information. + pub fn storage_value_change_block(key: &[u8]) -> Option { + storage::get_prefix(key) + .map(|change_block| { + let change_block: u64 = Decode::decode(&mut &change_block[..]) + .expect("failed to decode bock number from storage value prefix"); + As::sa(change_block) + }) + } + /// Calculate the current block's random seed. fn calculate_random() -> T::Hash { assert!(Self::block_number() > Zero::zero(), "Block number may never be zero"); @@ -195,27 +235,60 @@ impl Module { #[derive(Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(deny_unknown_fields)] -pub struct GenesisConfig(PhantomData); +pub struct GenesisConfig { + /// If true, all storage values are prepended with u64 block number and + /// storage_value_change_block is able to return Some(change_block). + pub use_block_number_prefix: bool, + /// Storage purge block interval (in blocks). Every storage_purge_interval + /// blocks, deleted values are purged from the storage. + pub storage_purge_interval: Option, + /// Minimal age (in blocks) of the deleted value to stay in storage until + /// it could be purged. + pub min_purged_value_age: Option, +} #[cfg(any(feature = "std", test))] impl Default for GenesisConfig { fn default() -> Self { - GenesisConfig(PhantomData) + GenesisConfig { + use_block_number_prefix: false, + storage_purge_interval: None, + min_purged_value_age: None, + } } } #[cfg(any(feature = "std", test))] impl primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> Result { + fn build_raw_storage(self) -> Result { use codec::Encode; - Ok(map![ + let mut storage: primitives::StorageMap = map![ Self::hash(&>::key_for(T::BlockNumber::zero())).to_vec() => [69u8; 32].encode(), Self::hash(>::key()).to_vec() => 1u64.encode(), Self::hash(>::key()).to_vec() => [69u8; 32].encode(), Self::hash(>::key()).to_vec() => [0u8; 32].encode(), Self::hash(>::key()).to_vec() => [0u8; 4].encode() - ]) + ]; + + if let Some(storage_purge_interval) = self.storage_purge_interval { + storage.insert( + Self::hash(>::key()).to_vec(), + storage_purge_interval.encode()); + } + + if let Some(min_purged_value_age) = self.min_purged_value_age { + storage.insert( + Self::hash(>::key()).to_vec(), + min_purged_value_age.encode()); + } + + if self.use_block_number_prefix { + storage.insert(runtime_io::PREFIX_LEN_KEY.to_vec(), 8u32.encode()); + storage.insert(runtime_io::PREFIX_KEY.to_vec(), [0u8; 8].encode()); + } + + Ok(storage) } } diff --git a/substrate/runtime/system/src/tests.rs b/substrate/runtime/system/src/tests.rs new file mode 100644 index 0000000000000..8c3b8ad46d9bd --- /dev/null +++ b/substrate/runtime/system/src/tests.rs @@ -0,0 +1,86 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// This file is part of Substrate Demo. + +// Substrate Demo is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate Demo is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate Demo. If not, see . + +//! Tests for the module. + +#![cfg(test)] + +use codec::Decode; +use primitives::{BuildStorage, testing::{Digest, Header}}; +use substrate_primitives::H256; +use runtime_io::{PREFIX_KEY, TestExternalities, PurgeFilterResult, + storage, clear_storage, purge_storage, with_externalities}; +use super::*; + +// Workaround for https://github.com/rust-lang/rust/issues/26925 . Remove when sorted. +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub struct Test; + +impl Trait for Test { + type Index = u64; + type BlockNumber = u64; + type Hash = H256; + type Hashing = ::primitives::traits::BlakeTwo256; + type Digest = Digest; + type AccountId = u64; + type Header = Header; +} + +fn new_test_ext(use_prefix: bool) -> TestExternalities { + let mut ext = if use_prefix { + GenesisConfig:: { + use_block_number_prefix: use_prefix, + storage_purge_interval: if use_prefix { Some(4) } else { None }, + min_purged_value_age: if use_prefix { Some(2) } else { None }, + } + } else { + Default::default() + }.build_storage().unwrap(); + ext.insert(b"dog".to_vec(), b"good".to_vec()); + ext +} + +#[test] +fn initialise_does_not_set_storage_prefix_by_default() { + with_externalities(&mut new_test_ext(false), || { + Module::::initialise(&1, &Default::default(), &Default::default()); + assert!(storage(PREFIX_KEY).is_none()); + }); +} + +#[test] +fn initialise_sets_storage_prefix_when_configured() { + with_externalities(&mut new_test_ext(true), || { + Module::::initialise(&1, &Default::default(), &Default::default()); + assert_eq!(Some(1u64), storage(PREFIX_KEY).and_then(|p| Decode::decode(&mut &p[..]))); + }); +} + +#[test] +fn key_is_not_scheduled_for_purge_if_created_and_deleted_in_the_same_block() { + with_externalities(&mut new_test_ext(true), || { + clear_storage(b"cat"); + assert_eq!(purge_storage(|_, _, _| PurgeFilterResult::Purge), 0); + }); +} + +#[test] +fn key_is_scheduled_for_purge() { + with_externalities(&mut new_test_ext(true), || { + clear_storage(b"dog"); + assert_eq!(purge_storage(|_, _, _| PurgeFilterResult::Purge), 1); + }); +} diff --git a/substrate/runtime/timestamp/src/lib.rs b/substrate/runtime/timestamp/src/lib.rs index 0d20bd6b58e97..150f78a90ed30 100644 --- a/substrate/runtime/timestamp/src/lib.rs +++ b/substrate/runtime/timestamp/src/lib.rs @@ -126,7 +126,7 @@ impl Default for GenesisConfig { #[cfg(any(feature = "std", test))] impl runtime_primitives::BuildStorage for GenesisConfig { - fn build_storage(self) -> ::std::result::Result { + fn build_raw_storage(self) -> ::std::result::Result { use codec::Encode; Ok(map![ Self::hash(>::key()).to_vec() => self.period.encode(), diff --git a/substrate/service/src/chain_spec.rs b/substrate/service/src/chain_spec.rs index 6ccd0545b4435..0e432de02b179 100644 --- a/substrate/service/src/chain_spec.rs +++ b/substrate/service/src/chain_spec.rs @@ -53,9 +53,19 @@ impl GenesisSource { } impl<'a, G: RuntimeGenesis> BuildStorage for &'a ChainSpec { + fn build_raw_storage(self) -> Result { + match self.genesis.resolve()? { + Genesis::Runtime(gc) => gc.build_raw_storage(), + Genesis::Raw(_) => + unreachable!("build_raw_storage is only called from build_storage; in build_storage we check that genesis is not Raw; qed"), + } + } + fn build_storage(self) -> Result { match self.genesis.resolve()? { + // values are not yet prefixed Genesis::Runtime(gc) => gc.build_storage(), + // raw means that values are already prefixed Genesis::Raw(map) => Ok(map.into_iter().map(|(k, v)| (k.0, v.0)).collect()), } } diff --git a/substrate/state-machine/Cargo.toml b/substrate/state-machine/Cargo.toml index 17b072c4c6687..f86ee41e99632 100644 --- a/substrate/state-machine/Cargo.toml +++ b/substrate/state-machine/Cargo.toml @@ -12,7 +12,7 @@ log = "0.3" parking_lot = "0.4" triehash = "0.1" -substrate-primitives = { path = "../primitives", version = "0.1.0" } +substrate-codec = { path = "../codec" } hashdb = { git = "https://github.com/paritytech/parity.git" } memorydb = { git = "https://github.com/paritytech/parity.git" } diff --git a/substrate/state-machine/src/ext.rs b/substrate/state-machine/src/ext.rs index 5a8ba096f5753..e845c30940ae9 100644 --- a/substrate/state-machine/src/ext.rs +++ b/substrate/state-machine/src/ext.rs @@ -14,10 +14,11 @@ // You should have received a copy of the GNU General Public License // along with Substrate. If not, see . -//! Conrete externalities implementation. +//! Concrete externalities implementation. -use std::{error, fmt}; +use std::{error, fmt, cell::RefCell, collections::HashMap}; use backend::Backend; +use keys_set::{Set as KeysSet, Storage as KeysSetStorage}; use {Externalities, OverlayedChanges}; /// Errors that can occur when interacting with the externalities. @@ -57,8 +58,13 @@ pub struct Ext<'a, B: 'a + Backend> { backend: &'a B, // The transaction necessary to commit to the backend. transaction: Option<(B::Transaction, [u8; 32])>, + // In-memory cache for frequently used values. + cache: RefCell, Option>>>, } +/// Implementation of keys set storage for Backend + OverlayedChanges. +struct ExtKeysSetStorage<'a, B: 'a>(&'a B, &'a mut OverlayedChanges); + impl<'a, B: 'a + Backend> Ext<'a, B> { /// Create a new `Ext` from overlayed changes and read-only backend pub fn new(overlay: &'a mut OverlayedChanges, backend: &'a B) -> Self { @@ -66,6 +72,7 @@ impl<'a, B: 'a + Backend> Ext<'a, B> { overlay, backend, transaction: None, + cache: Default::default(), } } @@ -83,22 +90,6 @@ impl<'a, B: 'a + Backend> Ext<'a, B> { } } -#[cfg(test)] -impl<'a, B: 'a + Backend> Ext<'a, B> { - pub fn storage_pairs(&self) -> Vec<(Vec, Vec)> { - use std::collections::HashMap; - - self.backend.pairs().iter() - .map(|&(ref k, ref v)| (k.to_vec(), Some(v.to_vec()))) - .chain(self.overlay.committed.clone().into_iter()) - .chain(self.overlay.prospective.clone().into_iter()) - .collect::>() - .into_iter() - .filter_map(|(k, maybe_val)| maybe_val.map(|val| (k, val))) - .collect() - } -} - impl<'a, B: 'a> Externalities for Ext<'a, B> where B: Backend { @@ -107,6 +98,23 @@ impl<'a, B: 'a> Externalities for Ext<'a, B> self.backend.storage(key).expect("Externalities not allowed to fail within runtime")) } + fn cached_storage(&self, key: &[u8]) -> Option> { + self.overlay.storage(key) + .map(|x| x.map(|x| x.to_vec())) + .unwrap_or_else(|| { + let value_from_cache = self.cache.borrow().get(key).cloned(); + match value_from_cache { + Some(value) => value, + None => { + let value = self.backend.storage(key) + .expect("Externalities not allowed to fail within runtime"); + self.cache.borrow_mut().insert(key.to_vec(), value.clone()); + value + }, + } + }) + } + fn exists_storage(&self, key: &[u8]) -> bool { match self.overlay.storage(key) { Some(x) => x.is_some(), @@ -114,19 +122,51 @@ impl<'a, B: 'a> Externalities for Ext<'a, B> } } + fn exists_previous_storage(&self, key: &[u8]) -> bool { + self.backend.exists_storage(key).expect("Externalities not allowed to fail within runtime") + } + fn place_storage(&mut self, key: Vec, value: Option>) { self.mark_dirty(); self.overlay.set_storage(key, value); } - fn clear_prefix(&mut self, prefix: &[u8]) { + fn place_prefix(&mut self, include_new_keys: bool, prefix: &[u8], value: Option>) { self.mark_dirty(); - self.overlay.clear_prefix(prefix); + if include_new_keys { + self.overlay.set_prefix(prefix, value.clone()); + } self.backend.for_keys_with_prefix(prefix, |key| { - self.overlay.set_storage(key.to_vec(), None); + self.overlay.set_storage(key.to_vec(), value.clone()); }); } + fn save_pefix_keys(&mut self, include_new_keys: bool, prefix: &[u8], set_prefix: &[u8]) { + // do not allow overlaps + // panic is safe here, since this should only be called from the runtime + assert!(!set_prefix.starts_with(prefix)); + + // collecting all keys from OverlayedChanges with filter couldn't + // take more memory than the OverlayedChanges itself + // => since we need to iterate OverlayedChanges + insert into OverlayedChanges + // => collect all affected OverlayedChanges keys before iterating backend + let mut overlayed_keys: Vec> = Vec::new(); + if include_new_keys { + self.overlay.for_keys_with_prefix(prefix, |key| overlayed_keys.push(key.to_vec())); + } + + // insert keys to the set + self.mark_dirty(); + { + let mut set_storage = ExtKeysSetStorage(self.backend, self.overlay); + let mut set = KeysSet::new(set_prefix, &mut set_storage); + for overlayed_key in overlayed_keys { + set.insert(&overlayed_key); + } + self.backend.for_keys_with_prefix(prefix, |key| set.insert(key)); + } + } + fn chain_id(&self) -> u64 { 42 } @@ -146,3 +186,157 @@ impl<'a, B: 'a> Externalities for Ext<'a, B> root } } + +impl<'a, B: 'a + Backend> KeysSetStorage for ExtKeysSetStorage<'a, B> { + fn read(&self, key: &[u8]) -> Option> { + self.1.storage(key).map(|x| x.map(|x| x.to_vec())).unwrap_or_else(|| + self.0.storage(key).expect("Externalities not allowed to fail within runtime")) + } + + fn set(&mut self, key: &[u8], value: &[u8]) { + self.1.set_storage(key.to_vec(), Some(value.to_vec())); + } + + fn clear(&mut self, key: &[u8]) { + self.1.set_storage(key.to_vec(), None); + } +} + +#[cfg(test)] +mod tests { + use backend::InMemory; + use testing::TestKeysSetStorage; + use TestExternalities; + use super::*; + + fn with_backend_based_ext(f: F) { + let backend: InMemory = vec![(b"doe".to_vec(), b"reindeer".to_vec()), + (b"dog".to_vec(), b"puppy".to_vec()), + (b"dogglesworth".to_vec(), b"cat".to_vec())].into_iter() + .collect::<::std::collections::HashMap<_, _>>().into(); + let mut changes = OverlayedChanges::default(); + let mut ext = Ext::new(&mut changes, &backend); + f(&mut ext); + } + + fn with_testing_ext(f: F) { + let mut ext = TestExternalities::new(); + ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); + ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); + ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); + f(&mut ext); + } + + #[test] + fn removal_with_place_prefix_works() { + fn do_test(ext: &mut Externalities) { + ext.place_prefix(true, b"dog", None); + assert!(ext.storage(b"doe").is_some()); + assert!(ext.storage(b"dog").is_none()); + assert!(ext.storage(b"dogglesworth").is_none()); + } + + with_backend_based_ext(do_test); + with_testing_ext(do_test); + } + + #[test] + fn ext_update_with_place_prefix_works() { + fn do_test(ext: &mut Externalities) { + ext.place_storage(b"doggy".to_vec(), Some(b"new_key".to_vec())); + ext.place_prefix(true, b"dog", Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"doe"), Some(b"reindeer".to_vec())); + assert_eq!(ext.storage(b"dog"), Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"dogglesworth"), Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"doggy"), Some(b"puma".to_vec())); + } + + with_backend_based_ext(do_test); + with_testing_ext(do_test); + } + + #[test] + fn ext_update_with_place_prefix_does_not_include_new_keys() { + fn do_test(ext: &mut Externalities) { + ext.place_storage(b"doggy".to_vec(), Some(b"new_key".to_vec())); + ext.place_prefix(false, b"dog", Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"doe"), Some(b"reindeer".to_vec())); + assert_eq!(ext.storage(b"dog"), Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"dogglesworth"), Some(b"puma".to_vec())); + assert_eq!(ext.storage(b"doggy"), Some(b"new_key".to_vec())); + } + + with_backend_based_ext(do_test); + } + + #[test] + fn testing_ext_save_prefix_keys_works() { + let mut ext = TestExternalities::new(); + ext.set_storage(b"doe".to_vec(), b"reindeer".to_vec()); + ext.set_storage(b"dog".to_vec(), b"puppy".to_vec()); + ext.set_storage(b"dogglesworth".to_vec(), b"cat".to_vec()); + + ext.save_pefix_keys(true, b"dog", b":deleted"); + + let mut set_storage = TestKeysSetStorage(&mut ext); + let mut set = KeysSet::new(b":deleted", &mut set_storage); + let mut saved_keys = ::std::collections::BTreeSet::new(); + set.retain(|_, key| { saved_keys.insert(key.to_vec()); false }); + assert_eq!(vec![b"dog".to_vec(), b"dogglesworth".to_vec()], + saved_keys.into_iter().collect::>()); + } + + #[test] + fn backend_ext_save_prefix_keys_works() { + let backend: InMemory = vec![ + (b"doe".to_vec(), b"reindeer".to_vec()), + (b"dog".to_vec(), b"puppy".to_vec()), + (b"dogglesworth".to_vec(), b"cat".to_vec())].into_iter() + .collect::<::std::collections::HashMap<_, _>>().into(); + let mut changes = OverlayedChanges::default(); + let mut ext = Ext::new(&mut changes, &backend); + + ext.save_pefix_keys(true, b"dog", b":deleted"); + + let mut set_storage = ExtKeysSetStorage(ext.backend, ext.overlay); + let mut set = KeysSet::new(b":deleted", &mut set_storage); + let mut saved_keys = ::std::collections::BTreeSet::new(); + set.retain(|_, key| { saved_keys.insert(key.to_vec()); false }); + assert_eq!(vec![b"dog".to_vec(), b"dogglesworth".to_vec()], + saved_keys.into_iter().collect::>()); + } + + #[test] + fn backend_ext_save_prefix_keys_does_not_include_new_keys() { + let backend: InMemory = vec![ + (b"doe".to_vec(), b"reindeer".to_vec()), + (b"dog".to_vec(), b"puppy".to_vec()), + (b"dogglesworth".to_vec(), b"cat".to_vec())].into_iter() + .collect::<::std::collections::HashMap<_, _>>().into(); + let mut changes = OverlayedChanges::default(); + let mut ext = Ext::new(&mut changes, &backend); + + ext.place_storage(b"dog".to_vec(), Some(b"old_key".to_vec())); + ext.place_storage(b"doggy".to_vec(), Some(b"new_key".to_vec())); + ext.save_pefix_keys(false, b"dog", b":deleted"); + + let mut set_storage = ExtKeysSetStorage(ext.backend, ext.overlay); + let mut set = KeysSet::new(b":deleted", &mut set_storage); + let mut saved_keys = ::std::collections::BTreeSet::new(); + set.retain(|_, key| { saved_keys.insert(key.to_vec()); false }); + assert_eq!(vec![b"dog".to_vec(), b"dogglesworth".to_vec()], + saved_keys.into_iter().collect::>()); + } + + #[test] + #[should_panic] + fn backend_ext_save_prefix_panics_when_used_wrong() { + with_backend_based_ext(|ext| ext.save_pefix_keys(false, b":de", b":deleted")); + } + + #[test] + #[should_panic] + fn testing_ext_save_prefix_panics_when_used_wrong() { + TestExternalities::new().save_pefix_keys(false, b":de", b":deleted") + } +} diff --git a/substrate/state-machine/src/lib.rs b/substrate/state-machine/src/lib.rs index 17fd85852b6ac..0f18f7f91cd18 100644 --- a/substrate/state-machine/src/lib.rs +++ b/substrate/state-machine/src/lib.rs @@ -27,6 +27,7 @@ extern crate log; extern crate ethereum_types; extern crate hashdb; extern crate memorydb; +extern crate substrate_codec as codec; extern crate triehash; extern crate patricia_trie; @@ -75,18 +76,18 @@ impl OverlayedChanges { self.prospective.insert(key, val); } - /// Removes all key-value pairs which keys share the given prefix. + /// Sets the value of all key-value pairs which keys share the given prefix. /// /// NOTE that this doesn't take place immediately but written into the prospective /// change set, and still can be reverted by [`discard_prospective`]. /// /// [`discard_prospective`]: #method.discard_prospective - fn clear_prefix(&mut self, prefix: &[u8]) { + fn set_prefix(&mut self, prefix: &[u8], value: Option>) { // Iterate over all prospective and mark all keys that share // the given prefix as removed (None). - for (key, value) in self.prospective.iter_mut() { + for (key, prosp_value) in self.prospective.iter_mut() { if key.starts_with(prefix) { - *value = None; + *prosp_value = value.clone(); } } @@ -94,11 +95,20 @@ impl OverlayedChanges { // NOTE that we are making changes in the prospective change set. for key in self.committed.keys() { if key.starts_with(prefix) { - self.prospective.insert(key.to_owned(), None); + self.prospective.insert(key.to_owned(), value.clone()); } } } + /// Runs the given closure for all keys that are starting with given prefix. + fn for_keys_with_prefix(&self, prefix: &[u8], f: F) { + self.prospective.keys() + .chain(self.committed.keys()) + .filter(|key| key.starts_with(prefix)) + .map(|k| &**k) + .for_each(f); + } + /// Discard prospective changes to state. pub fn discard_prospective(&mut self) { self.prospective.clear(); @@ -153,6 +163,29 @@ pub trait Externalities { /// Read storage of current contract being called. fn storage(&self, key: &[u8]) -> Option>; + /// Read (preferably from the cache, if enabled) storage of current contract being called. + fn cached_storage(&self, key: &[u8]) -> Option> { + self.storage(key) + } + + /// Read storage of current contract being called and strips the prefix from the value. + /// If you're reading storage values from substrate, you should use this method or else + /// storage() call could return wrong value if runtime uses storage prefixes. + fn stripped_storage(&self, key: &[u8]) -> Option> { + self.storage(key) + .and_then(|value| { + let prefix_len = self.storage(self::prefix::PREFIX_LEN_KEY); + match self::prefix::strip_prefix(prefix_len, value) { + Ok((_, stripped_value)) => stripped_value, + Err(error) => { + warn!("Failed to parse storage prefix length: {}", + error); + None + }, + } + }) + } + /// Set storage entry `key` of current contract being called (effective immediately). fn set_storage(&mut self, key: Vec, value: Vec) { self.place_storage(key, Some(value)); @@ -163,17 +196,44 @@ pub trait Externalities { self.place_storage(key.to_vec(), None); } - /// Clear a storage entry (`key`) of current contract being called (effective immediately). + /// Checks if storage entry (`key`) of current contract being called exists. fn exists_storage(&self, key: &[u8]) -> bool { self.storage(key).is_some() } + /// Checks if storage entry (`key`) of current contract being called existed at the moment + /// when Externalities were created. + fn exists_previous_storage(&self, key: &[u8]) -> bool; + + /// Set storage entries which keys are start with the given prefix to the given value. + /// If include_new_keys is true, all keys that have been created since the moment + /// of Externalities creation are also updated. Otherwise - only keys that + /// pre-existed before are affected. + fn set_prefix(&mut self, include_new_keys: bool, prefix: &[u8], value: Vec) { + self.place_prefix(include_new_keys, prefix, Some(value)); + } + /// Clear storage entries which keys are start with the given prefix. - fn clear_prefix(&mut self, prefix: &[u8]); + fn clear_prefix(&mut self, prefix: &[u8]) { + self.place_prefix(true, prefix, None); + } + + /// Save all keys, starting with the given prefix to the keys set with given prefix. + /// Prefix should not include set_prefix. + /// If include_new_keys is true, all keys that have been created since the moment + /// of Externalities creation are also saved. Otherwise - only save keys that + /// pre-existed before. + fn save_pefix_keys(&mut self, include_new_keys: bool, prefix: &[u8], set_prefix: &[u8]); /// Set or clear a storage entry (`key`) of current contract being called (effective immediately). fn place_storage(&mut self, key: Vec, value: Option>); + /// Set or clear a storage entry with given `prefix` of current contract being called (effective immediately). + /// If include_new_keys is true, all keys that have been created since the moment + /// of Externalities creation are also updated. Otherwise - only keys that + /// pre-existed before are affected. + fn place_prefix(&mut self, include_new_keys: bool, prefix: &[u8], value: Option>); + /// Get the identity of the chain. fn chain_id(&self) -> u64; @@ -295,7 +355,8 @@ pub fn execute_using_consensus_failure_handler< let strategy: ExecutionStrategy = (&manager).into(); // make a copy. - let code = ext::Ext::new(overlay, backend).storage(b":code") + let code = ext::Ext::new(overlay, backend) + .stripped_storage(b":code") .ok_or_else(|| Box::new(ExecutionError::CodeEntryDoesNotExist) as Box)? .to_vec(); @@ -414,6 +475,18 @@ pub fn execution_proof_check( execute(&backend, overlay, exec, method, call_data, ExecutionStrategy::NativeWhenPossible) } +/// Module with prefix-related utilities which are used by both runtime && substrate. +mod prefix { + include!("../../runtime-io/src/prefix_shared.rs"); +} + +/// Module with KeysSet structure which is used by both runtime && substrate. +mod keys_set { + use codec::{Decode, Encode}; + + include!("../../runtime-io/src/keys_set.rs"); +} + #[cfg(test)] mod tests { use super::*; @@ -607,4 +680,15 @@ mod tests { ], ); } + + #[test] + fn stripped_storage_works() { + let mut ext = TestExternalities::new(); + ext.set_storage(b"key".to_vec(), b"val".to_vec()); + assert_eq!(ext.storage(b"key"), Some(b"val".to_vec())); + + ext.set_storage(prefix::PREFIX_LEN_KEY.to_vec(), vec![0, 0, 0, 0, 4, 0, 0, 0]); + ext.set_storage(b"key".to_vec(), b"0000val".to_vec()); + assert_eq!(ext.stripped_storage(b"key"), Some(b"val".to_vec())); + } } diff --git a/substrate/state-machine/src/testing.rs b/substrate/state-machine/src/testing.rs index 7b85b523cd47a..208cd75f8a155 100644 --- a/substrate/state-machine/src/testing.rs +++ b/substrate/state-machine/src/testing.rs @@ -17,17 +17,27 @@ //! Test implementation for Externalities. use std::collections::HashMap; -use super::Externalities; use triehash::trie_root; +use keys_set::{Set as KeysSet, Storage as KeysSetStorage}; +use Externalities; /// Simple HashMap based Externalities impl. pub type TestExternalities = HashMap, Vec>; +/// Implementation of keys set storage for TestExternalities. +pub(crate) struct TestKeysSetStorage<'a>(pub &'a mut TestExternalities); + impl Externalities for TestExternalities { fn storage(&self, key: &[u8]) -> Option> { self.get(key).map(|x| x.to_vec()) } + fn exists_previous_storage(&self, key: &[u8]) -> bool { + // we do not really need this in testing, since storage in testing is + // created for single block only => just work as exists_storage() here + self.get(key).is_some() + } + fn place_storage(&mut self, key: Vec, maybe_value: Option>) { match maybe_value { Some(value) => { self.insert(key, value); } @@ -35,10 +45,38 @@ impl Externalities for TestExternalities { } } - fn clear_prefix(&mut self, prefix: &[u8]) { - self.retain(|key, _| - !key.starts_with(prefix) - ) + fn place_prefix(&mut self, _include_new_keys: bool, prefix: &[u8], value: Option>) { + match value { + None => self.retain(|key, _| !key.starts_with(prefix)), + Some(value) => { + for (key, data_value) in self.iter_mut() { + if key.starts_with(prefix) { + *data_value = value.clone(); + } + } + }, + } + } + + fn save_pefix_keys(&mut self, _include_new_keys: bool, prefix: &[u8], set_prefix: &[u8]) { + // in test implementation, prefix overlap won't lead to inifinite execution + // but other implementations could suffer + // => do not allow overlaps + // panic is safe here, since this should only be called from the runtime + assert!(!set_prefix.starts_with(prefix)); + + // it is safe to collect here, since TestExternalities are used in tests only + let keys_to_save: Vec<_> = self.keys() + .filter(|key| key.starts_with(prefix)) + .cloned() + .collect(); + + // insert keys to the set + let mut set_storage = TestKeysSetStorage(self); + let mut set = KeysSet::new(set_prefix, &mut set_storage); + for key in keys_to_save { + set.insert(&key); + } } fn chain_id(&self) -> u64 { 42 } @@ -48,6 +86,20 @@ impl Externalities for TestExternalities { } } +impl<'a> KeysSetStorage for TestKeysSetStorage<'a> { + fn read(&self, key: &[u8]) -> Option> { + self.0.storage(key) + } + + fn set(&mut self, key: &[u8], value: &[u8]) { + self.0.place_storage(key.to_vec(), Some(value.to_vec())); + } + + fn clear(&mut self, key: &[u8]) { + self.0.place_storage(key.to_vec(), None); + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/substrate/test-client/src/client_ext.rs b/substrate/test-client/src/client_ext.rs index be1e9dfe55026..068c3ef39d1c2 100644 --- a/substrate/test-client/src/client_ext.rs +++ b/substrate/test-client/src/client_ext.rs @@ -27,7 +27,7 @@ use {Backend, Executor}; /// Extension trait for a test client. pub trait TestClient { - /// Crates new client instance for tests. + /// Creates new client instance for tests. fn new_for_tests() -> Self; /// Justify and import block to the chain. diff --git a/substrate/test-runtime/wasm/Cargo.lock b/substrate/test-runtime/wasm/Cargo.lock index e78284183ae39..8baf51c06d051 100644 --- a/substrate/test-runtime/wasm/Cargo.lock +++ b/substrate/test-runtime/wasm/Cargo.lock @@ -757,7 +757,7 @@ dependencies = [ "memorydb 0.1.1 (git+https://github.com/paritytech/parity.git)", "parking_lot 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "patricia-trie 0.1.0 (git+https://github.com/paritytech/parity.git)", - "substrate-primitives 0.1.0", + "substrate-codec 0.1.0", "triehash 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm b/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm index 6356848ee723a..07ccf76dde7c9 100644 Binary files a/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm and b/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.compact.wasm differ diff --git a/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.wasm b/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.wasm index 3dbbf9b9f3ba1..6b96de3db4963 100755 Binary files a/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.wasm and b/substrate/test-runtime/wasm/target/wasm32-unknown-unknown/release/substrate_test_runtime.wasm differ